{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:38.839592Z",
     "iopub.status.busy": "2023-10-23T07:01:38.839329Z",
     "iopub.status.idle": "2023-10-23T07:01:39.167676Z",
     "shell.execute_reply": "2023-10-23T07:01:39.166763Z",
     "shell.execute_reply.started": "2023-10-23T07:01:38.839567Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:39.170448Z",
     "iopub.status.busy": "2023-10-23T07:01:39.169595Z",
     "iopub.status.idle": "2023-10-23T07:01:42.568158Z",
     "shell.execute_reply": "2023-10-23T07:01:42.567068Z",
     "shell.execute_reply.started": "2023-10-23T07:01:39.170412Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "from torch.optim import AdamW\n",
    "from torch.utils.data import Dataset, DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:42.569775Z",
     "iopub.status.busy": "2023-10-23T07:01:42.569396Z",
     "iopub.status.idle": "2023-10-23T07:01:52.251928Z",
     "shell.execute_reply": "2023-10-23T07:01:52.251057Z",
     "shell.execute_reply.started": "2023-10-23T07:01:42.569751Z"
    }
   },
   "outputs": [],
   "source": [
    "import transformers\n",
    "from transformers import GPT2LMHeadModel, GPT2Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:52.254344Z",
     "iopub.status.busy": "2023-10-23T07:01:52.254047Z",
     "iopub.status.idle": "2023-10-23T07:01:52.606992Z",
     "shell.execute_reply": "2023-10-23T07:01:52.606142Z",
     "shell.execute_reply.started": "2023-10-23T07:01:52.254309Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:52.609267Z",
     "iopub.status.busy": "2023-10-23T07:01:52.608592Z",
     "iopub.status.idle": "2023-10-23T07:01:52.614202Z",
     "shell.execute_reply": "2023-10-23T07:01:52.613147Z",
     "shell.execute_reply.started": "2023-10-23T07:01:52.609232Z"
    }
   },
   "outputs": [],
   "source": [
    "def free_memory():\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:52.616246Z",
     "iopub.status.busy": "2023-10-23T07:01:52.615595Z",
     "iopub.status.idle": "2023-10-23T07:01:53.365885Z",
     "shell.execute_reply": "2023-10-23T07:01:53.364997Z",
     "shell.execute_reply.started": "2023-10-23T07:01:52.616192Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>качество плохое пошив ужасный (горловина напер...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Товар отдали другому человеку, я не получила п...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ужасная синтетика! Тонкая, ничего общего с пре...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>товар не пришел, продавец продлил защиту без м...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Кофточка голая синтетика, носить не возможно.</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Очень глубокие проймы</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Я недовольна заказом.Я вот одного не понимаю п...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>заказала размер s на от 64,об 94,начнем с того...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Заказ я сделала в июле. С тех пор посылка отсл...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Ужасное качество товара!</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  качество плохое пошив ужасный (горловина напер...  negative\n",
       "1  Товар отдали другому человеку, я не получила п...  negative\n",
       "2  Ужасная синтетика! Тонкая, ничего общего с пре...  negative\n",
       "3  товар не пришел, продавец продлил защиту без м...  negative\n",
       "4      Кофточка голая синтетика, носить не возможно.  negative\n",
       "5                              Очень глубокие проймы  negative\n",
       "6  Я недовольна заказом.Я вот одного не понимаю п...  negative\n",
       "7  заказала размер s на от 64,об 94,начнем с того...  negative\n",
       "8  Заказ я сделала в июле. С тех пор посылка отсл...  negative\n",
       "9                           Ужасное качество товара!  negative"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews = pd.read_csv('data/reviews.csv.zip', sep='\\t')\n",
    "reviews.drop_duplicates(subset=['review'], inplace=True)\n",
    "reviews.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:53.367445Z",
     "iopub.status.busy": "2023-10-23T07:01:53.367074Z",
     "iopub.status.idle": "2023-10-23T07:01:53.650422Z",
     "shell.execute_reply": "2023-10-23T07:01:53.649470Z",
     "shell.execute_reply.started": "2023-10-23T07:01:53.367416Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGdCAYAAAASUnlxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0bUlEQVR4nO3de3xNd77/8XcSyU6CHdckMoKcapHWXbH1pkQ2Mk61pocyLQY9PJK25Lg0c5SgfaQ11dLSmjmdis5hRs1MdYqS3RhUpS4hbi2jhjHzYEfrFqGSSNbvj/6yjl2XSuxI5ft6Ph55PKy1Puu7vmvZ35131mXvAMuyLAEAABggsLo7AAAAcKsQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxqhV3R2oTmVlZTp27Jjq1q2rgICA6u4OAAC4AZZl6dy5c4qJiVFgYMXO4RgdfI4dO6bY2Njq7gYAAKiEf/7zn2ratGmF1jE6+NStW1fSdwfO6XT6rd2SkhJlZWUpMTFRwcHBfmsXwI1jHALVqyrHYEFBgWJjY+3f4xVhdPApv7zldDr9HnzCw8PldDp5wwWqCeMQqF63YgxW5jYVbm4GAADGIPgAAABjVCj4vP3222rXrp19acjlcunjjz+2l1+8eFHJyclq2LCh6tSpo0GDBik/P9+njaNHjyopKUnh4eGKjIzUpEmTdOnSJZ+a9evXq1OnTnI4HGrZsqUyMzOv6MuCBQvUokULhYaGqlu3btq6dWtFdgUAABioQsGnadOmevnll5Wbm6vt27erV69eeuSRR7Rv3z5J0oQJE/TRRx9p+fLl2rBhg44dO6bHHnvMXr+0tFRJSUkqLi7W5s2btXjxYmVmZmratGl2zeHDh5WUlKSHH35YeXl5Gj9+vEaPHq21a9faNcuWLVNqaqqmT5+uHTt2qH379nK73Tpx4sTNHg8AAFCTWTepfv361jvvvGOdOXPGCg4OtpYvX24v+/LLLy1JVk5OjmVZlrV69WorMDDQ8nq9ds3bb79tOZ1Oq6ioyLIsy5o8ebJ19913+2xj8ODBltvttqe7du1qJScn29OlpaVWTEyMlZGRUaG+nz171pJknT17tkLr/ZDi4mJrxYoVVnFxsV/bBXDjGIdA9arKMXgzv78r/VRXaWmpli9frvPnz8vlcik3N1clJSVKSEiwa1q3bq1mzZopJydH3bt3V05Ojtq2bauoqCi7xu12a9y4cdq3b586duyonJwcnzbKa8aPHy9JKi4uVm5urtLS0uzlgYGBSkhIUE5OznX7XFRUpKKiInu6oKBA0nd3npeUlFT2UFyhvC1/tgmgYhiHQPWqyjF4M21WOPjs2bNHLpdLFy9eVJ06dfTBBx8oPj5eeXl5CgkJUb169Xzqo6Ki5PV6JUler9cn9JQvL192vZqCggJ9++23On36tEpLS69as3///uv2PSMjQzNmzLhiflZWlsLDw3945yvI4/H4vU0AFcM4BKpXVYzBCxcuVHrdCgefVq1aKS8vT2fPntUf//hHDR8+XBs2bKh0B26ltLQ0paam2tPlH4CUmJjo98/x8Xg86tOnD58fAlQTxiFQvapyDJZfsamMCgefkJAQtWzZUpLUuXNnbdu2TfPmzdPgwYNVXFysM2fO+Jz1yc/PV3R0tCQpOjr6iqevyp/6urzm+0+C5efny+l0KiwsTEFBQQoKCrpqTXkb1+JwOORwOK6YHxwcXCVvjFXVLoAbxzgEqldVjMGbae+mP8enrKxMRUVF6ty5s4KDg5WdnW0vO3DggI4ePSqXyyVJcrlc2rNnj8/TVx6PR06nU/Hx8XbN5W2U15S3ERISos6dO/vUlJWVKTs7264BAAC4mgqd8UlLS1O/fv3UrFkznTt3TkuXLtX69eu1du1aRUREaNSoUUpNTVWDBg3kdDr1zDPPyOVyqXv37pKkxMRExcfH68knn9Ts2bPl9Xo1depUJScn22dixo4dq/nz52vy5Mn6xS9+oXXr1un999/XqlWr7H6kpqZq+PDh6tKli7p27aq5c+fq/PnzGjlypB8PDQAAqGkqFHxOnDihp556SsePH1dERITatWuntWvXqk+fPpKk119/XYGBgRo0aJCKiorkdrv11ltv2esHBQVp5cqVGjdunFwul2rXrq3hw4dr5syZdk1cXJxWrVqlCRMmaN68eWratKneeecdud1uu2bw4MH6+uuvNW3aNHm9XnXo0EFr1qy54oZnAACAywVYlmVVdyeqS0FBgSIiInT27Fm/39y8evVq9e/fn3sLgGrCOASqV1WOwZv5/c13dQEAAGNU+gMMAQDArdPi+VU/XPQj4giyNLtrdffiSgSfKnRP+loVlQZUdzdu2JGXk6q7CwAAVCkudQEAAGMQfAAAgDEIPgAAwBgEHwAAYAxubgZQo/GQAYDLccYHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGKNCwScjI0P33nuv6tatq8jISA0cOFAHDhzwqenZs6cCAgJ8fsaOHetTc/ToUSUlJSk8PFyRkZGaNGmSLl265FOzfv16derUSQ6HQy1btlRmZuYV/VmwYIFatGih0NBQdevWTVu3bq3I7gAAAMNUKPhs2LBBycnJ+vzzz+XxeFRSUqLExESdP3/ep27MmDE6fvy4/TN79mx7WWlpqZKSklRcXKzNmzdr8eLFyszM1LRp0+yaw4cPKykpSQ8//LDy8vI0fvx4jR49WmvXrrVrli1bptTUVE2fPl07duxQ+/bt5Xa7deLEicoeCwAAUMPVqkjxmjVrfKYzMzMVGRmp3NxcPfjgg/b88PBwRUdHX7WNrKwsffHFF/rkk08UFRWlDh06aNasWZoyZYrS09MVEhKihQsXKi4uTnPmzJEktWnTRps2bdLrr78ut9stSXrttdc0ZswYjRw5UpK0cOFCrVq1Su+++66ef/75iuwWAAAwRIWCz/edPXtWktSgQQOf+UuWLNH//u//Kjo6WgMGDNALL7yg8PBwSVJOTo7atm2rqKgou97tdmvcuHHat2+fOnbsqJycHCUkJPi06Xa7NX78eElScXGxcnNzlZaWZi8PDAxUQkKCcnJyrtnfoqIiFRUV2dMFBQWSpJKSEpWUlFTiCFxdeVuOQMtvbd4K/jwGQHVjHKKmcQTdXq/l8rFXFa/pm2mz0sGnrKxM48eP13333ad77rnHnj906FA1b95cMTEx2r17t6ZMmaIDBw7oz3/+syTJ6/X6hB5J9rTX671uTUFBgb799ludPn1apaWlV63Zv3//NfuckZGhGTNmXDE/KyvLDmb+NKtLmd/brEqrV6+u7i4Afsc4RE0xu2t196ByPB6P39u8cOFCpdetdPBJTk7W3r17tWnTJp/5Tz/9tP3vtm3bqkmTJurdu7cOHTqkO+64o9Id9Ye0tDSlpqba0wUFBYqNjVViYqKcTqfftlNSUiKPx6MXtgeqqCzAb+1Wtb3p7uruAuA3jEPUNPekr/3hoh8RR6ClWV3K1KdPHwUHB/u17fIrNpVRqeCTkpKilStXauPGjWratOl1a7t16yZJ+uqrr3THHXcoOjr6iqev8vPzJcm+Lyg6Otqed3mN0+lUWFiYgoKCFBQUdNWaa91bJEkOh0MOh+OK+cHBwX7/T5GkorIAFZXePm+4VXEMgOrGOERNcTu9ji9XFb9jb6a9Cj3VZVmWUlJS9MEHH2jdunWKi4v7wXXy8vIkSU2aNJEkuVwu7dmzx+fpK4/HI6fTqfj4eLsmOzvbpx2PxyOXyyVJCgkJUefOnX1qysrKlJ2dbdcAAAB8X4XO+CQnJ2vp0qX68MMPVbduXfuenIiICIWFhenQoUNaunSp+vfvr4YNG2r37t2aMGGCHnzwQbVr106SlJiYqPj4eD355JOaPXu2vF6vpk6dquTkZPtszNixYzV//nxNnjxZv/jFL7Ru3Tq9//77WrVqld2X1NRUDR8+XF26dFHXrl01d+5cnT9/3n7KCwAA4PsqFHzefvttSd99SOHlFi1apBEjRigkJESffPKJHUJiY2M1aNAgTZ061a4NCgrSypUrNW7cOLlcLtWuXVvDhw/XzJkz7Zq4uDitWrVKEyZM0Lx589S0aVO988479qPskjR48GB9/fXXmjZtmrxerzp06KA1a9ZcccMzAABAuQoFH8u6/qN0sbGx2rBhww+207x58x98cqFnz57auXPndWtSUlKUkpLyg9sDAACQ+K4uAABgEIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgjAoFn4yMDN17772qW7euIiMjNXDgQB04cMCn5uLFi0pOTlbDhg1Vp04dDRo0SPn5+T41R48eVVJSksLDwxUZGalJkybp0qVLPjXr169Xp06d5HA41LJlS2VmZl7RnwULFqhFixYKDQ1Vt27dtHXr1orsDgAAMEyFgs+GDRuUnJyszz//XB6PRyUlJUpMTNT58+ftmgkTJuijjz7S8uXLtWHDBh07dkyPPfaYvby0tFRJSUkqLi7W5s2btXjxYmVmZmratGl2zeHDh5WUlKSHH35YeXl5Gj9+vEaPHq21a9faNcuWLVNqaqqmT5+uHTt2qH379nK73Tpx4sTNHA8AAFCDBViWZVV25a+//lqRkZHasGGDHnzwQZ09e1aNGzfW0qVL9bOf/UyStH//frVp00Y5OTnq3r27Pv74Y/30pz/VsWPHFBUVJUlauHChpkyZoq+//lohISGaMmWKVq1apb1799rbGjJkiM6cOaM1a9ZIkrp166Z7771X8+fPlySVlZUpNjZWzzzzjJ5//vkb6n9BQYEiIiJ09uxZOZ3Oyh6GK5SUlGj16tWavDVIRaUBfmu3qh15Oam6uwD4DeMQNU2L51dVdxcqxBFkaXbXUvXv31/BwcF+bftmfn/XupkNnz17VpLUoEEDSVJubq5KSkqUkJBg17Ru3VrNmjWzg09OTo7atm1rhx5JcrvdGjdunPbt26eOHTsqJyfHp43ymvHjx0uSiouLlZubq7S0NHt5YGCgEhISlJOTc83+FhUVqaioyJ4uKCiQ9N0bZElJSSWPwpXK23IEVjpTVgt/HgOgujEOUdM4gm6v13L52KuK1/TNtFnp4FNWVqbx48frvvvu0z333CNJ8nq9CgkJUb169Xxqo6Ki5PV67ZrLQ0/58vJl16spKCjQt99+q9OnT6u0tPSqNfv3779mnzMyMjRjxowr5mdlZSk8PPwG9rpiZnUp83ubVWn16tXV3QXA7xiHqClmd63uHlSOx+Pxe5sXLlyo9LqVDj7Jycnau3evNm3aVOmN32ppaWlKTU21pwsKChQbG6vExES/X+ryeDx6YXugispun1Pse9Pd1d0FwG8Yh6hp7klf+8NFPyKOQEuzupSpT58+VXKpq7IqFXxSUlK0cuVKbdy4UU2bNrXnR0dHq7i4WGfOnPE565Ofn6/o6Gi75vtPX5U/9XV5zfefBMvPz5fT6VRYWJiCgoIUFBR01ZryNq7G4XDI4XBcMT84ONjv/ymSVFQWcFvdW1AVxwCoboxD1BS30+v4clXxO/Zm2qvQU12WZSklJUUffPCB1q1bp7i4OJ/lnTt3VnBwsLKzs+15Bw4c0NGjR+VyuSRJLpdLe/bs8Xn6yuPxyOl0Kj4+3q65vI3ymvI2QkJC1LlzZ5+asrIyZWdn2zUAAADfV6EzPsnJyVq6dKk+/PBD1a1b174nJyIiQmFhYYqIiNCoUaOUmpqqBg0ayOl06plnnpHL5VL37t0lSYmJiYqPj9eTTz6p2bNny+v1aurUqUpOTrbPxowdO1bz58/X5MmT9Ytf/ELr1q3T+++/r1Wr/u+O9tTUVA0fPlxdunRR165dNXfuXJ0/f14jR47017EBAAA1TIWCz9tvvy1J6tmzp8/8RYsWacSIEZKk119/XYGBgRo0aJCKiorkdrv11ltv2bVBQUFauXKlxo0bJ5fLpdq1a2v48OGaOXOmXRMXF6dVq1ZpwoQJmjdvnpo2bap33nlHbvf/XfsePHiwvv76a02bNk1er1cdOnTQmjVrrrjhGQAAoFyFgs+NfORPaGioFixYoAULFlyzpnnz5j/45ELPnj21c+fO69akpKQoJSXlB/sEAAAg8V1dAADAIAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGBUOPhs3btSAAQMUExOjgIAArVixwmf5iBEjFBAQ4PPTt29fn5pTp05p2LBhcjqdqlevnkaNGqXCwkKfmt27d+uBBx5QaGioYmNjNXv27Cv6snz5crVu3VqhoaFq27atVq9eXdHdAQAABqlw8Dl//rzat2+vBQsWXLOmb9++On78uP3z+9//3mf5sGHDtG/fPnk8Hq1cuVIbN27U008/bS8vKChQYmKimjdvrtzcXP3qV79Senq6fvOb39g1mzdv1hNPPKFRo0Zp586dGjhwoAYOHKi9e/dWdJcAAIAhalV0hX79+qlfv37XrXE4HIqOjr7qsi+//FJr1qzRtm3b1KVLF0nSm2++qf79++vVV19VTEyMlixZouLiYr377rsKCQnR3Xffrby8PL322mt2QJo3b5769u2rSZMmSZJmzZolj8ej+fPna+HChRXdLQAAYIAKB58bsX79ekVGRqp+/frq1auXXnzxRTVs2FCSlJOTo3r16tmhR5ISEhIUGBioLVu26NFHH1VOTo4efPBBhYSE2DVut1uvvPKKTp8+rfr16ysnJ0epqak+23W73VdcertcUVGRioqK7OmCggJJUklJiUpKSvyx63Z7kuQItPzW5q3gz2MAVDfGIWoaR9Dt9VouH3tV8Zq+mTb9Hnz69u2rxx57THFxcTp06JB++ctfql+/fsrJyVFQUJC8Xq8iIyN9O1Grlho0aCCv1ytJ8nq9iouL86mJioqyl9WvX19er9eed3lNeRtXk5GRoRkzZlwxPysrS+Hh4ZXa3+uZ1aXM721WJe6RQk3EOERNMbtrdfegcjwej9/bvHDhQqXX9XvwGTJkiP3vtm3bql27drrjjju0fv169e7d29+bq5C0tDSfs0QFBQWKjY1VYmKinE6n37ZTUlIij8ejF7YHqqgswG/tVrW96e7q7gLgN4xD1DT3pK+t7i5UiCPQ0qwuZerTp4+Cg4P92nb5FZvKqJJLXZf7t3/7NzVq1EhfffWVevfurejoaJ04ccKn5tKlSzp16pR9X1B0dLTy8/N9asqnf6jmWvcWSd/de+RwOK6YHxwc7Pf/FEkqKgtQUent84ZbFccAqG6MQ9QUt9Pr+HJV8Tv2Ztqr8s/x+de//qWTJ0+qSZMmkiSXy6UzZ84oNzfXrlm3bp3KysrUrVs3u2bjxo0+1/A8Ho9atWql+vXr2zXZ2dk+2/J4PHK5XFW9SwAA4DZV4eBTWFiovLw85eXlSZIOHz6svLw8HT16VIWFhZo0aZI+//xzHTlyRNnZ2XrkkUfUsmVLud3fnb5t06aN+vbtqzFjxmjr1q367LPPlJKSoiFDhigmJkaSNHToUIWEhGjUqFHat2+fli1bpnnz5vlcpnruuee0Zs0azZkzR/v371d6erq2b9+ulJQUPxwWAABQE1U4+Gzfvl0dO3ZUx44dJUmpqanq2LGjpk2bpqCgIO3evVv//u//rrvuukujRo1S586d9emnn/pcYlqyZIlat26t3r17q3///rr//vt9PqMnIiJCWVlZOnz4sDp37qz/+q//0rRp03w+66dHjx5aunSpfvOb36h9+/b64x//qBUrVuiee+65meMBAABqsArf49OzZ09Z1rUfqVu79odvvmrQoIGWLl163Zp27drp008/vW7N448/rscff/wHtwcAACDxXV0AAMAgBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYFQ4+Gzdu1IABAxQTE6OAgACtWLHCZ7llWZo2bZqaNGmisLAwJSQk6ODBgz41p06d0rBhw+R0OlWvXj2NGjVKhYWFPjW7d+/WAw88oNDQUMXGxmr27NlX9GX58uVq3bq1QkND1bZtW61evbqiuwMAAAxS4eBz/vx5tW/fXgsWLLjq8tmzZ+uNN97QwoULtWXLFtWuXVtut1sXL160a4YNG6Z9+/bJ4/Fo5cqV2rhxo55++ml7eUFBgRITE9W8eXPl5ubqV7/6ldLT0/Wb3/zGrtm8ebOeeOIJjRo1Sjt37tTAgQM1cOBA7d27t6K7BAAADFGroiv069dP/fr1u+oyy7I0d+5cTZ06VY888ogk6b333lNUVJRWrFihIUOG6Msvv9SaNWu0bds2denSRZL05ptvqn///nr11VcVExOjJUuWqLi4WO+++65CQkJ09913Ky8vT6+99podkObNm6e+fftq0qRJkqRZs2bJ4/Fo/vz5WrhwYaUOBgAAqNkqHHyu5/Dhw/J6vUpISLDnRUREqFu3bsrJydGQIUOUk5OjevXq2aFHkhISEhQYGKgtW7bo0UcfVU5Ojh588EGFhITYNW63W6+88opOnz6t+vXrKycnR6mpqT7bd7vdV1x6u1xRUZGKiors6YKCAklSSUmJSkpKbnb3beVtOQItv7V5K/jzGADVjXGImsYRdHu9lsvHXlW8pm+mTb8GH6/XK0mKiorymR8VFWUv83q9ioyM9O1ErVpq0KCBT01cXNwVbZQvq1+/vrxe73W3czUZGRmaMWPGFfOzsrIUHh5+I7tYIbO6lPm9zarEPVKoiRiHqClmd63uHlSOx+Pxe5sXLlyo9Lp+DT4/dmlpaT5niQoKChQbG6vExEQ5nU6/baekpEQej0cvbA9UUVmA39qtanvT3dXdBcBvGIeoae5JX1vdXagQR6ClWV3K1KdPHwUHB/u17fIrNpXh1+ATHR0tScrPz1eTJk3s+fn5+erQoYNdc+LECZ/1Ll26pFOnTtnrR0dHKz8/36emfPqHasqXX43D4ZDD4bhifnBwsN//UySpqCxARaW3zxtuVRwDoLoxDlFT3E6v48tVxe/Ym2nPr5/jExcXp+joaGVnZ9vzCgoKtGXLFrlcLkmSy+XSmTNnlJuba9esW7dOZWVl6tatm12zceNGn2t4Ho9HrVq1Uv369e2ay7dTXlO+HQAAgO+rcPApLCxUXl6e8vLyJH13Q3NeXp6OHj2qgIAAjR8/Xi+++KL+8pe/aM+ePXrqqacUExOjgQMHSpLatGmjvn37asyYMdq6das+++wzpaSkaMiQIYqJiZEkDR06VCEhIRo1apT27dunZcuWad68eT6XqZ577jmtWbNGc+bM0f79+5Wenq7t27crJSXl5o8KAACokSp8qWv79u16+OGH7enyMDJ8+HBlZmZq8uTJOn/+vJ5++mmdOXNG999/v9asWaPQ0FB7nSVLliglJUW9e/dWYGCgBg0apDfeeMNeHhERoaysLCUnJ6tz585q1KiRpk2b5vNZPz169NDSpUs1depU/fKXv9Sdd96pFStW6J577qnUgQAAADVfhYNPz549ZVnXfqQuICBAM2fO1MyZM69Z06BBAy1duvS622nXrp0+/fTT69Y8/vjjevzxx6/fYQAAgP+P7+oCAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADG8HvwSU9PV0BAgM9P69at7eUXL15UcnKyGjZsqDp16mjQoEHKz8/3aePo0aNKSkpSeHi4IiMjNWnSJF26dMmnZv369erUqZMcDodatmypzMxMf+8KAACoYarkjM/dd9+t48eP2z+bNm2yl02YMEEfffSRli9frg0bNujYsWN67LHH7OWlpaVKSkpScXGxNm/erMWLFyszM1PTpk2zaw4fPqykpCQ9/PDDysvL0/jx4zV69GitXbu2KnYHAADUELWqpNFatRQdHX3F/LNnz+q3v/2tli5dql69ekmSFi1apDZt2ujzzz9X9+7dlZWVpS+++EKffPKJoqKi1KFDB82aNUtTpkxRenq6QkJCtHDhQsXFxWnOnDmSpDZt2mjTpk16/fXX5Xa7q2KXAABADVAlwefgwYOKiYlRaGioXC6XMjIy1KxZM+Xm5qqkpEQJCQl2bevWrdWsWTPl5OSoe/fuysnJUdu2bRUVFWXXuN1ujRs3Tvv27VPHjh2Vk5Pj00Z5zfjx46/br6KiIhUVFdnTBQUFkqSSkhKVlJT4Yc9ltydJjkDLb23eCv48BkB1YxyipnEE3V6v5fKxVxWv6Ztp0+/Bp1u3bsrMzFSrVq10/PhxzZgxQw888ID27t0rr9erkJAQ1atXz2edqKgoeb1eSZLX6/UJPeXLy5ddr6agoEDffvutwsLCrtq3jIwMzZgx44r5WVlZCg8Pr9T+Xs+sLmV+b7MqrV69urq7APgd4xA1xeyu1d2DyvF4PH5v88KFC5Ve1+/Bp1+/fva/27Vrp27duql58+Z6//33rxlIbpW0tDSlpqba0wUFBYqNjVViYqKcTqfftlNSUiKPx6MXtgeqqCzAb+1Wtb3pXCZEzcE4RE1zT/rtdR+rI9DSrC5l6tOnj4KDg/3advkVm8qokktdl6tXr57uuusuffXVV+rTp4+Ki4t15swZn7M++fn59j1B0dHR2rp1q08b5U99XV7z/SfB8vPz5XQ6rxuuHA6HHA7HFfODg4P9/p8iSUVlASoqvX3ecKviGADVjXGImuJ2eh1frip+x95Me1X+OT6FhYU6dOiQmjRpos6dOys4OFjZ2dn28gMHDujo0aNyuVySJJfLpT179ujEiRN2jcfjkdPpVHx8vF1zeRvlNeVtAAAAXI3fg8/EiRO1YcMGHTlyRJs3b9ajjz6qoKAgPfHEE4qIiNCoUaOUmpqqv/71r8rNzdXIkSPlcrnUvXt3SVJiYqLi4+P15JNPateuXVq7dq2mTp2q5ORk+2zN2LFj9fe//12TJ0/W/v379dZbb+n999/XhAkT/L07AACgBvH7pa5//etfeuKJJ3Ty5Ek1btxY999/vz7//HM1btxYkvT6668rMDBQgwYNUlFRkdxut9566y17/aCgIK1cuVLjxo2Ty+VS7dq1NXz4cM2cOdOuiYuL06pVqzRhwgTNmzdPTZs21TvvvMOj7AAA4Lr8Hnz+8Ic/XHd5aGioFixYoAULFlyzpnnz5j/4ZEPPnj21c+fOSvURAACYie/qAgAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxrjtg8+CBQvUokULhYaGqlu3btq6dWt1dwkAAPxI3dbBZ9myZUpNTdX06dO1Y8cOtW/fXm63WydOnKjurgEAgB+h2zr4vPbaaxozZoxGjhyp+Ph4LVy4UOHh4Xr33Xeru2sAAOBHqFZ1d6CyiouLlZubq7S0NHteYGCgEhISlJOTc9V1ioqKVFRUZE+fPXtWknTq1CmVlJT4rW8lJSW6cOGCapUEqrQswG/tVrWTJ09WdxcAv2Ecoqapdel8dXehQmqVWbpwoUwnT55UcHCwX9s+d+6cJMmyrIr3y689uYW++eYblZaWKioqymd+VFSU9u/ff9V1MjIyNGPGjCvmx8XFVUkfbzeN5lR3DwAwDlGTDK3i9s+dO6eIiIgKrXPbBp/KSEtLU2pqqj1dVlamU6dOqWHDhgoI8N9fhAUFBYqNjdU///lPOZ1Ov7UL4MYxDoHqVZVj0LIsnTt3TjExMRVe97YNPo0aNVJQUJDy8/N95ufn5ys6Ovqq6zgcDjkcDp959erVq6ouyul08oYLVDPGIVC9qmoMVvRMT7nb9ubmkJAQde7cWdnZ2fa8srIyZWdny+VyVWPPAADAj9Vte8ZHklJTUzV8+HB16dJFXbt21dy5c3X+/HmNHDmyursGAAB+hG7r4DN48GB9/fXXmjZtmrxerzp06KA1a9ZcccPzreZwODR9+vQrLqsBuHUYh0D1+rGOwQCrMs+CAQAA3IZu23t8AAAAKorgAwAAjEHwAQAAxiD4VLP09HR16NChursBoJLWr1+vgIAAnTlzprq7Avwo3OiYaNGihebOnXtL+nQ5gs8tFBAQoBUrVvjMmzhxos9nEQGoWkeOHFFAQIDy8vKquytAjdSjRw8dP37c/oDBzMzMq35Y8LZt2/T000/f4t7d5o+z1wR16tRRnTp1qrsbAL6nuLhYISEh1d0N4LYTEhJyzW9QuFzjxo1vQW+uZMQZn549e+rZZ5/V5MmT1aBBA0VHRys9Pd1efubMGY0ePVqNGzeW0+lUr169tGvXLp82XnzxRUVGRqpu3boaPXq0nn/+eZ9LVNu2bVOfPn3UqFEjRURE6KGHHtKOHTvs5S1atJAkPfroowoICLCnL7/UlZWVpdDQ0CtODz733HPq1auXPb1p0yY98MADCgsLU2xsrJ599lmdP397fWsvzHSzY/HQoUN65JFHFBUVpTp16ujee+/VJ5984rONq51ZrVevnjIzMyX935cSd+zYUQEBAerZs6ckacSIERo4cKBeeuklxcTEqFWrVpKk3/3ud+rSpYvq1q2r6OhoDR06VCdOnPDvgQFusZ49eyolJUUpKSmKiIhQo0aN9MILL9jfdn769Gk99dRTql+/vsLDw9WvXz8dPHjQXv8f//iHBgwYoPr166t27dq6++67tXr1akm+l7rWr1+vkSNH6uzZswoICFBAQIA95i+/1DV06FANHjzYp48lJSVq1KiR3nvvPUnffTtDRkaG4uLiFBYWpvbt2+uPf/xjhffdiOAjSYsXL1bt2rW1ZcsWzZ49WzNnzpTH45EkPf744zpx4oQ+/vhj5ebmqlOnTurdu7dOnTolSVqyZIleeuklvfLKK8rNzVWzZs309ttv+7R/7tw5DR8+XJs2bdLnn3+uO++8U/3799e5c+ckfReMJGnRokU6fvy4PX253r17q169evrTn/5kzystLdWyZcs0bNgwSd+98fft21eDBg3S7t27tWzZMm3atEkpKSn+P2hAFbiZsVhYWKj+/fsrOztbO3fuVN++fTVgwAAdPXr0hre/detWSdInn3yi48eP689//rO9LDs7WwcOHJDH49HKlSslfffmO2vWLO3atUsrVqzQkSNHNGLECD8dDaD6LF68WLVq1dLWrVs1b948vfbaa3rnnXckffeHwPbt2/WXv/xFOTk5sixL/fv3V0lJiSQpOTlZRUVF2rhxo/bs2aNXXnnlqlcvevTooblz58rpdOr48eM6fvy4Jk6ceEXdsGHD9NFHH6mwsNCet3btWl24cEGPPvqoJCkjI0PvvfeeFi5cqH379mnChAn6+c9/rg0bNlRsxy0DPPTQQ9b999/vM+/ee++1pkyZYn366aeW0+m0Ll686LP8jjvusH79619blmVZ3bp1s5KTk32W33fffVb79u2vuc3S0lKrbt261kcffWTPk2R98MEHPnXTp0/3aee5556zevXqZU+vXbvWcjgc1unTpy3LsqxRo0ZZTz/9tE8bn376qRUYGGh9++231+wP8GNws2Pxau6++27rzTfftKevNs4iIiKsRYsWWZZlWYcPH7YkWTt37vSpGT58uBUVFWUVFRVddx+2bdtmSbLOnTtnWZZl/fWvf7Uk2WMUuB089NBDVps2bayysjJ73pQpU6w2bdpYf/vb3yxJ1meffWYv++abb6ywsDDr/ffftyzLstq2bWulp6dfte3vj4lFixZZERERV9Q1b97cev311y3LsqySkhKrUaNG1nvvvWcvf+KJJ6zBgwdblmVZFy9etMLDw63Nmzf7tDFq1CjriSeeqNC+G3PGp127dj7TTZo00YkTJ7Rr1y4VFhaqYcOG9v02derU0eHDh3Xo0CFJ0oEDB9S1a1ef9b8/nZ+frzFjxujOO+9URESEnE6nCgsLK/SXqPRd6l2/fr2OHTsm6buzTUlJSfaNYbt27VJmZqZPX91ut8rKynT48OEKbQuoDjczFgsLCzVx4kS1adNG9erVU506dfTll19WeJxdS9u2ba+4ryc3N1cDBgxQs2bNVLduXT300EOS5LdtAtWle/fuCggIsKddLpcOHjyoL774QrVq1VK3bt3sZQ0bNlSrVq305ZdfSpKeffZZvfjii7rvvvs0ffp07d69+6b6UqtWLf3Hf/yHlixZIkk6f/68PvzwQ/tqx1dffaULFy6oT58+Pu8P7733nv3+cMPbuqme3kaCg4N9pgMCAlRWVqbCwkI1adJE69evv2Kdq92Ffi3Dhw/XyZMnNW/ePDVv3lwOh0Mul0vFxcUV6ue9996rO+64Q3/4wx80btw4ffDBB/a9CdJ3b/z/+Z//qWefffaKdZs1a1ahbQHV4WbG4sSJE+XxePTqq6+qZcuWCgsL089+9jOfcRYQEGDfp1Cu/PT8D6ldu7bP9Pnz5+V2u+V2u7VkyRI1btxYR48eldvtrvDYBmqS0aNHy+12a9WqVcrKylJGRobmzJmjZ555ptJtDhs2TA899JBOnDghj8ejsLAw9e3bV5LsS2CrVq3ST37yE5/1KvpdYMYEn2vp1KmTvF6vatWqZd9w/H2tWrXStm3b9NRTT9nzvn+Pzmeffaa33npL/fv3lyT985//1DfffONTExwcrNLS0h/s07Bhw7RkyRI1bdpUgYGBSkpK8unvF198oZYtW97oLgK3hRsZi5999plGjBhhX/MvLCzUkSNHfGoaN26s48eP29MHDx7UhQsX7OnyMzo3Mhb379+vkydP6uWXX1ZsbKwkafv27RXZLeBHa8uWLT7T5fenxsfH69KlS9qyZYt69OghSTp58qQOHDig+Ph4uz42NlZjx47V2LFjlZaWpv/5n/+5avAJCQm5ofHWo0cPxcbGatmyZfr444/1+OOP238oxcfHy+Fw6OjRo/ZZ18oy5lLXtSQkJMjlcmngwIHKysrSkSNHtHnzZv33f/+3/Qb3zDPP6Le//a0WL16sgwcP6sUXX9Tu3bt9ThHeeeed+t3vfqcvv/xSW7Zs0bBhwxQWFuazrRYtWig7O1ter1enT5++Zp+GDRumHTt26KWXXtLPfvYznzQ7ZcoUbd68WSkpKcrLy9PBgwf14YcfcnMzbns3MhbvvPNO/fnPf1ZeXp527dqloUOHqqyszKedXr16af78+dq5c6e2b9+usWPH+pxlioyMVFhYmNasWaP8/HydPXv2mn1q1qyZQkJC9Oabb+rvf/+7/vKXv2jWrFlVcwCAW+zo0aNKTU3VgQMH9Pvf/15vvvmmnnvuOd1555165JFHNGbMGG3atEm7du3Sz3/+c/3kJz/RI488IkkaP3681q5dq8OHD2vHjh3661//qjZt2lx1Oy1atFBhYaGys7P1zTff+Pwh8n1Dhw7VwoUL5fF47MtcklS3bl1NnDhREyZM0OLFi3Xo0CHt2LFDb775phYvXlyh/TY++AQEBGj16tV68MEHNXLkSN11110aMmSI/vGPfygqKkrSd0EkLS1NEydOVKdOnXT48GGNGDFCoaGhdju//e1vdfr0aXXq1ElPPvmknn32WUVGRvpsa86cOfJ4PIqNjVXHjh2v2aeWLVuqa9eu2r17t89/vPTd/REbNmzQ3/72Nz3wwAPq2LGjpk2bppiYGD8eFeDWu5Gx+Nprr6l+/frq0aOHBgwYILfbrU6dOvm0M2fOHMXGxuqBBx7Q0KFDNXHiRIWHh9vLa9WqpTfeeEO//vWvFRMTY7+RX03jxo2VmZmp5cuXKz4+Xi+//LJeffXVqjkAwC321FNP6dtvv1XXrl2VnJys5557zv5AwUWLFqlz58766U9/KpfLJcuytHr1avuPiNLSUiUnJ6tNmzbq27ev7rrrLr311ltX3U6PHj00duxYDR48WI0bN9bs2bOv2adhw4bpiy++0E9+8hPdd999PstmzZqlF154QRkZGfZ2V61aZX9ExY0KsL5/MRw3pE+fPoqOjtbvfve76u4KAAAV0rNnT3Xo0KFavjKiuhl/j8+NuHDhghYuXCi3262goCD9/ve/1yeffGJ/9ggAALg9EHxuQPkp+JdeekkXL15Uq1at9Kc//UkJCQnV3TUAAFABXOoCAADGMP7mZgAAYA6CDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgjP8HaHMWNKrk8iUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "reviews.sentiment.hist()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:53.651768Z",
     "iopub.status.busy": "2023-10-23T07:01:53.651500Z",
     "iopub.status.idle": "2023-10-23T07:01:53.656438Z",
     "shell.execute_reply": "2023-10-23T07:01:53.655535Z",
     "shell.execute_reply.started": "2023-10-23T07:01:53.651744Z"
    }
   },
   "outputs": [],
   "source": [
    "sentiment_map = {\n",
    "    'negative': 0,\n",
    "    'neautral': 1,\n",
    "    'positive': 2\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:53.660002Z",
     "iopub.status.busy": "2023-10-23T07:01:53.659742Z",
     "iopub.status.idle": "2023-10-23T07:01:53.727237Z",
     "shell.execute_reply": "2023-10-23T07:01:53.726393Z",
     "shell.execute_reply.started": "2023-10-23T07:01:53.659979Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>качество плохое пошив ужасный (горловина напер...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Товар отдали другому человеку, я не получила п...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ужасная синтетика! Тонкая, ничего общего с пре...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>товар не пришел, продавец продлил защиту без м...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Кофточка голая синтетика, носить не возможно.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Очень глубокие проймы</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Я недовольна заказом.Я вот одного не понимаю п...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>заказала размер s на от 64,об 94,начнем с того...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Заказ я сделала в июле. С тех пор посылка отсл...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Ужасное качество товара!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  sentiment\n",
       "0  качество плохое пошив ужасный (горловина напер...          0\n",
       "1  Товар отдали другому человеку, я не получила п...          0\n",
       "2  Ужасная синтетика! Тонкая, ничего общего с пре...          0\n",
       "3  товар не пришел, продавец продлил защиту без м...          0\n",
       "4      Кофточка голая синтетика, носить не возможно.          0\n",
       "5                              Очень глубокие проймы          0\n",
       "6  Я недовольна заказом.Я вот одного не понимаю п...          0\n",
       "7  заказала размер s на от 64,об 94,начнем с того...          0\n",
       "8  Заказ я сделала в июле. С тех пор посылка отсл...          0\n",
       "9                           Ужасное качество товара!          0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews.sentiment = reviews.sentiment.apply(lambda x: sentiment_map[x])\n",
    "reviews.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:53.728630Z",
     "iopub.status.busy": "2023-10-23T07:01:53.728303Z",
     "iopub.status.idle": "2023-10-23T07:01:53.758273Z",
     "shell.execute_reply": "2023-10-23T07:01:53.757305Z",
     "shell.execute_reply.started": "2023-10-23T07:01:53.728599Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = ('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:53.759744Z",
     "iopub.status.busy": "2023-10-23T07:01:53.759449Z",
     "iopub.status.idle": "2023-10-23T07:01:53.767374Z",
     "shell.execute_reply": "2023-10-23T07:01:53.766575Z",
     "shell.execute_reply.started": "2023-10-23T07:01:53.759721Z"
    }
   },
   "outputs": [],
   "source": [
    "pretrained_model_name_or_path = 'sberbank-ai/rugpt3small_based_on_gpt2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:01:53.768752Z",
     "iopub.status.busy": "2023-10-23T07:01:53.768436Z",
     "iopub.status.idle": "2023-10-23T07:02:46.458137Z",
     "shell.execute_reply": "2023-10-23T07:02:46.457140Z",
     "shell.execute_reply.started": "2023-10-23T07:01:53.768722Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74c92fd8819c4ceb93f26057994c1d9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/608 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb242db3668f4c3eb556e996abaf919a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/551M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ebc0ca2163349ff8933d9c9fb0d4f33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.71M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4748c41149e4936adb7fbb17114348f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/1.27M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gpt = GPT2LMHeadModel.from_pretrained(pretrained_model_name_or_path)\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model_name_or_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:46.459649Z",
     "iopub.status.busy": "2023-10-23T07:02:46.459347Z",
     "iopub.status.idle": "2023-10-23T07:02:46.467919Z",
     "shell.execute_reply": "2023-10-23T07:02:46.465712Z",
     "shell.execute_reply.started": "2023-10-23T07:02:46.459623Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPT2LMHeadModel(\n",
       "  (transformer): GPT2Model(\n",
       "    (wte): Embedding(50264, 768)\n",
       "    (wpe): Embedding(2048, 768)\n",
       "    (drop): Dropout(p=0.1, inplace=False)\n",
       "    (h): ModuleList(\n",
       "      (0-11): 12 x GPT2Block(\n",
       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (act): NewGELUActivation()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=768, out_features=50264, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:46.469600Z",
     "iopub.status.busy": "2023-10-23T07:02:46.469255Z",
     "iopub.status.idle": "2023-10-23T07:02:47.827454Z",
     "shell.execute_reply": "2023-10-23T07:02:47.826440Z",
     "shell.execute_reply.started": "2023-10-23T07:02:46.469569Z"
    }
   },
   "outputs": [],
   "source": [
    "for param in gpt.parameters():\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:47.828845Z",
     "iopub.status.busy": "2023-10-23T07:02:47.828590Z",
     "iopub.status.idle": "2023-10-23T07:02:47.842744Z",
     "shell.execute_reply": "2023-10-23T07:02:47.841856Z",
     "shell.execute_reply.started": "2023-10-23T07:02:47.828824Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_labels = reviews.sentiment.nunique()\n",
    "num_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:47.844173Z",
     "iopub.status.busy": "2023-10-23T07:02:47.843831Z",
     "iopub.status.idle": "2023-10-23T07:02:52.920748Z",
     "shell.execute_reply": "2023-10-23T07:02:52.919839Z",
     "shell.execute_reply.started": "2023-10-23T07:02:47.844142Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPT2LMHeadModel(\n",
       "  (transformer): GPT2Model(\n",
       "    (wte): Embedding(50264, 768)\n",
       "    (wpe): Embedding(2048, 768)\n",
       "    (drop): Dropout(p=0.1, inplace=False)\n",
       "    (h): ModuleList(\n",
       "      (0-11): 12 x GPT2Block(\n",
       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (act): NewGELUActivation()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt.lm_head = nn.Linear(gpt.config.hidden_size, num_labels)\n",
    "gpt = gpt.to(device)\n",
    "gpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:52.928272Z",
     "iopub.status.busy": "2023-10-23T07:02:52.928007Z",
     "iopub.status.idle": "2023-10-23T07:02:52.946490Z",
     "shell.execute_reply": "2023-10-23T07:02:52.945593Z",
     "shell.execute_reply.started": "2023-10-23T07:02:52.928249Z"
    }
   },
   "outputs": [],
   "source": [
    "X = reviews.review\n",
    "y = reviews.sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:52.947742Z",
     "iopub.status.busy": "2023-10-23T07:02:52.947502Z",
     "iopub.status.idle": "2023-10-23T07:02:52.971990Z",
     "shell.execute_reply": "2023-10-23T07:02:52.971084Z",
     "shell.execute_reply.started": "2023-10-23T07:02:52.947721Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:52.973371Z",
     "iopub.status.busy": "2023-10-23T07:02:52.973073Z",
     "iopub.status.idle": "2023-10-23T07:02:52.978423Z",
     "shell.execute_reply": "2023-10-23T07:02:52.977551Z",
     "shell.execute_reply.started": "2023-10-23T07:02:52.973347Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad>\n"
     ]
    }
   ],
   "source": [
    "tokenizer.pad_token_id = 0\n",
    "print(tokenizer.pad_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:52.979842Z",
     "iopub.status.busy": "2023-10-23T07:02:52.979581Z",
     "iopub.status.idle": "2023-10-23T07:02:52.989643Z",
     "shell.execute_reply": "2023-10-23T07:02:52.988838Z",
     "shell.execute_reply.started": "2023-10-23T07:02:52.979812Z"
    }
   },
   "outputs": [],
   "source": [
    "class SentimentAnalysisDataset(Dataset):\n",
    "    def __init__(self, X, y, tokenizer=None, num_labels=3):\n",
    "        self.X = tokenizer(X.tolist(), \n",
    "                           truncation=True, \n",
    "                           padding=True,\n",
    "                           return_tensors=\"pt\",\n",
    "                           max_length=2048).to(device)\n",
    "        self.y = torch.tensor(y.to_numpy(), dtype=torch.int64).to(device)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return self.X.input_ids[idx], self.X.attention_mask[idx], self.y[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:02:52.991221Z",
     "iopub.status.busy": "2023-10-23T07:02:52.990651Z",
     "iopub.status.idle": "2023-10-23T07:04:38.846230Z",
     "shell.execute_reply": "2023-10-23T07:04:38.845437Z",
     "shell.execute_reply.started": "2023-10-23T07:02:52.991196Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = SentimentAnalysisDataset(X_train, y_train, tokenizer)\n",
    "test_dataset = SentimentAnalysisDataset(X_test, y_test, tokenizer)\n",
    "\n",
    "free_memory()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:04:38.847829Z",
     "iopub.status.busy": "2023-10-23T07:04:38.847481Z",
     "iopub.status.idle": "2023-10-23T07:04:38.852271Z",
     "shell.execute_reply": "2023-10-23T07:04:38.851393Z",
     "shell.execute_reply.started": "2023-10-23T07:04:38.847799Z"
    }
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "num_epochs = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:04:38.857203Z",
     "iopub.status.busy": "2023-10-23T07:04:38.856898Z",
     "iopub.status.idle": "2023-10-23T07:04:38.865576Z",
     "shell.execute_reply": "2023-10-23T07:04:38.864762Z",
     "shell.execute_reply.started": "2023-10-23T07:04:38.857179Z"
    }
   },
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:04:38.878045Z",
     "iopub.status.busy": "2023-10-23T07:04:38.877728Z",
     "iopub.status.idle": "2023-10-23T07:04:38.887916Z",
     "shell.execute_reply": "2023-10-23T07:04:38.886892Z",
     "shell.execute_reply.started": "2023-10-23T07:04:38.878021Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = AdamW(gpt.lm_head.parameters(), lr=0.001)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T15:38:46.707562Z",
     "iopub.status.busy": "2023-10-23T15:38:46.706888Z",
     "iopub.status.idle": "2023-10-23T15:38:46.715925Z",
     "shell.execute_reply": "2023-10-23T15:38:46.714886Z",
     "shell.execute_reply.started": "2023-10-23T15:38:46.707527Z"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate_model(model, data_loader):\n",
    "    ground_truth = []\n",
    "    predictions = []\n",
    "\n",
    "    model.eval()\n",
    "    for input_ids, attention_mask, labels in data_loader:\n",
    "        input_ids = input_ids.to(device)\n",
    "        attention_mask = attention_mask.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        logits = torch.mean(outputs.logits, dim=1)\n",
    "        _, indices = torch.max(logits, 1)\n",
    "        predictions.extend(indices.tolist())\n",
    "        ground_truth.extend(labels.tolist())\n",
    "        del input_ids\n",
    "        del attention_mask\n",
    "        del labels\n",
    "        del outputs\n",
    "        del logits\n",
    "        del indices\n",
    "        free_memory()\n",
    "    accuracy = accuracy_score(ground_truth, predictions)\n",
    "    del predictions\n",
    "    del ground_truth\n",
    "    free_memory()\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:04:38.889352Z",
     "iopub.status.busy": "2023-10-23T07:04:38.889048Z",
     "iopub.status.idle": "2023-10-23T07:04:38.899971Z",
     "shell.execute_reply": "2023-10-23T07:04:38.899234Z",
     "shell.execute_reply.started": "2023-10-23T07:04:38.889328Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_model(model, train_loader, criterion, optimizer, num_epochs):\n",
    "    loss_history = []\n",
    "    train_history = []\n",
    "    \n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        model.train()\n",
    "        \n",
    "        loss_accum = 0\n",
    "        correct_samples = 0\n",
    "        total_samples = 0\n",
    "        \n",
    "        for i_step, (input_ids, attention_mask, y) in enumerate(train_loader):\n",
    "            input_ids = input_ids.to(device)\n",
    "            attention_mask = attention_mask.to(device)\n",
    "            y = y.to(device)\n",
    "            outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "            logits = torch.mean(outputs.logits, dim=1)\n",
    "            optimizer.zero_grad()\n",
    "            loss = criterion(logits, y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            _, indices = torch.max(logits, 1)\n",
    "            correct_samples += torch.sum(indices == y)\n",
    "            total_samples += y.shape[0]\n",
    "            loss_accum += loss\n",
    "            \n",
    "            del outputs\n",
    "            del logits\n",
    "            del indices\n",
    "            del input_ids\n",
    "            del attention_mask\n",
    "            del y\n",
    "            free_memory()\n",
    "            \n",
    "        ave_loss = loss_accum / i_step\n",
    "        train_accuracy = float(correct_samples) / total_samples\n",
    "\n",
    "        loss_history.append(float(ave_loss))\n",
    "        train_history.append(train_accuracy)\n",
    "        \n",
    "        free_memory()\n",
    "\n",
    "        print(\"Average loss: %f, Train accuracy: %f\" % (ave_loss, train_accuracy))\n",
    "        \n",
    "    return loss_history, train_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T07:04:38.901310Z",
     "iopub.status.busy": "2023-10-23T07:04:38.900975Z",
     "iopub.status.idle": "2023-10-23T14:06:09.432044Z",
     "shell.execute_reply": "2023-10-23T14:06:09.430884Z",
     "shell.execute_reply.started": "2023-10-23T07:04:38.901287Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss: 0.673101, Train accuracy: 0.696590\n",
      "Average loss: 0.641609, Train accuracy: 0.709989\n",
      "Average loss: 0.634727, Train accuracy: 0.714198\n",
      "Average loss: 0.630941, Train accuracy: 0.716975\n",
      "Average loss: 0.628454, Train accuracy: 0.717175\n"
     ]
    }
   ],
   "source": [
    "loss_history, train_history = train_model(gpt, \n",
    "                                          train_loader, \n",
    "                                          criterion, \n",
    "                                          optimizer, num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T14:06:09.491276Z",
     "iopub.status.busy": "2023-10-23T14:06:09.490856Z",
     "iopub.status.idle": "2023-10-23T14:06:09.745229Z",
     "shell.execute_reply": "2023-10-23T14:06:09.744000Z",
     "shell.execute_reply.started": "2023-10-23T14:06:09.491240Z"
    }
   },
   "outputs": [],
   "source": [
    "free_memory()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T14:06:09.747534Z",
     "iopub.status.busy": "2023-10-23T14:06:09.746574Z",
     "iopub.status.idle": "2023-10-23T14:06:10.063812Z",
     "shell.execute_reply": "2023-10-23T14:06:10.062798Z",
     "shell.execute_reply.started": "2023-10-23T14:06:09.747479Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f5ba9067010>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGzCAYAAAAMr0ziAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABRB0lEQVR4nO3deVhU9f4H8PfMwAz7LgiCgIrmCgrCVSu9vzRLK7VNTRO3FnPNMvGWW3m10spbWloqWmpatllumVtlJgJqbuCOqCwqy7DOwMz5/TEwMuyDwDkD79fzzAOcOefM58tE8/Z8lyMTBEEAERERkYTJxS6AiIiIqCYMLERERCR5DCxEREQkeQwsREREJHkMLERERCR5DCxEREQkeQwsREREJHkMLERERCR5DCxEREQkeQwsRNQoAgICMHbsWLHLICILxcBCRHWyfv16yGQyxMbGVvp8v3790KVLl3t6jZ07d2LBggX3dA4iahoYWIioUSQmJuKLL74w65idO3di4cKFDVQREVkSBhYiahQqlQrW1tZilwG9Xo/CwkKxyyAiMzGwEFGjKD+GpaioCAsXLkRQUBBsbGzg7u6O+++/H3v37gUAjB07FitXrgQAyGQy46NUXl4eXnvtNfj5+UGlUqFDhw5YtmwZyt+AXiaTYcqUKdi0aRM6d+4MlUqFXbt2ISAgAEOGDKlQZ2FhIZydnfHSSy81wG+BiOrKSuwCiMiyZWdn4/bt2xW2FxUVVXvcggULsGTJEkycOBHh4eFQq9WIjY1FfHw8BgwYgJdeegk3b97E3r178dVXX5kcKwgCnnjiCRw4cAATJkxASEgI9uzZg1mzZuHGjRv46KOPTPbfv38/vvnmG0yZMgUeHh4IDAzE6NGj8f777yMjIwNubm7GfX/++Weo1WqMHj36Hn4rRFTvBCKiOoiOjhYAVPvo3LmzcX9/f38hMjLS+HNwcLAwePDgal9j8uTJQmX/m/rxxx8FAMKiRYtMtj/99NOCTCYTLl68aNwGQJDL5cKZM2dM9k1MTBQACJ999pnJ9ieeeEIICAgQ9Hp9jb8DImo87BIionuycuVK7N27t8KjW7du1R7n4uKCM2fO4MKFC2a/5s6dO6FQKDBt2jST7a+99hoEQcCuXbtMtvft2xedOnUy2da+fXtERERg06ZNxm0ZGRnYtWsXRo0aZdL9RETiY5cQEd2T8PBwhIWFVdju6upaaVdRqbfffhtDhgxB+/bt0aVLFzzyyCN4/vnnaww6AJCUlAQfHx84OjqabO/YsaPx+bICAwMrPc+YMWMwZcoUJCUlwd/fH99++y2Kiorw/PPP11gDETUuXmEhIlE8+OCDuHTpEtatW4cuXbpgzZo16NGjB9asWVPvr2Vra1vp9hEjRsDa2tp4lWXjxo0ICwtDhw4d6r0GIro3DCxEJBo3NzeMGzcOX3/9NZKTk9GtWzeTheKq6pbx9/fHzZs3kZOTY7I9ISHB+HxtX3/w4MHYtGkTkpKScPjwYV5dIZIoBhYiEsWdO3dMfnZwcEC7du2g0WiM2+zt7QEAWVlZJvsOGjQIOp0OK1asMNn+0UcfQSaT4dFHH611Hc8//zzOnj2LWbNmQaFQYMSIEWa2hIgaA8ewEJEoOnXqhH79+iE0NBRubm6IjY3Ftm3bMGXKFOM+oaGhAIBp06Zh4MCBxkDx+OOP49///jfefPNNXL16FcHBwfj111/x008/YcaMGWjbtm2t6xg8eDDc3d3x7bff4tFHH4Wnp2e9t5WI7h0DCxGJYtq0adi+fTt+/fVXaDQa+Pv7Y9GiRZg1a5ZxnyeffBJTp07Fli1bsHHjRgiCgBEjRkAul2P79u2YN28etm7diujoaAQEBGDp0qV47bXXzKpDqVRi+PDh+PTTT9kdRCRhMkEotywkEVEz8+qrr2Lt2rVITU2FnZ2d2OUQUSU4hoWImrXCwkJs3LgRTz31FMMKkYSxS4iImqX09HT89ttv2LZtG+7cuYPp06eLXRIRVYOBhYiapbNnz2LUqFHw9PTExx9/jJCQELFLIqJqcAwLERERSR7HsBAREZHkMbAQERGR5DWZMSx6vR43b96Eo6Mj77JKRERkIQRBQE5ODnx8fCCXV30dpckElps3b8LPz0/sMoiIiKgOkpOT4evrW+XzTSawlN5mPjk5GU5OTiJXQ0RERLWhVqvh5+dn/ByvSp0Cy8qVK7F06VKkpqYiODgYn3zyCcLDwyvdt1+/fjh06FCF7YMGDcKOHTtQVFSEt956Czt37sTly5fh7OyM/v37491334WPj0+tayrtBnJycmJgISIisjA1Decwe9Dt1q1bMXPmTMyfPx/x8fEIDg7GwIEDkZ6eXun+33//PVJSUoyP06dPQ6FQ4JlnngEA5OfnIz4+HnPnzkV8fDy+//57JCYm4oknnjC3NCIiImqizF6HJSIiAj179jTe1l2v18PPzw9Tp05FVFRUjccvX74c8+bNQ0pKivHW8eUdO3YM4eHhSEpKQuvWrWtVl1qthrOzM7Kzs3mFhYiIyELU9vPbrCssWq0WcXFx6N+//90TyOXo378/jhw5UqtzrF27FiNGjKgyrABAdnY2ZDIZXFxcqtxHo9FArVabPIiIiKhpMmsMy+3bt6HT6eDl5WWy3cvLCwkJCTUeHxMTg9OnT2Pt2rVV7lNYWIjZs2dj5MiR1SatJUuWYOHChbUvHoBOp0NRUZFZx1DTpVAoYGVlxWnwREQWoFFnCa1duxZdu3atcoBuUVERnn32WQiCgM8++6zac82ZMwczZ840/lw6yrgqubm5uH79OngnAirLzs4O3t7eUCqVYpdCRETVMCuweHh4QKFQIC0tzWR7WloaWrZsWe2xeXl52LJlC95+++1Kny8NK0lJSdi/f3+N41BUKhVUKlWt6tbpdLh+/Trs7OzQokUL/ouaIAgCtFotbt26hStXriAoKKjaBYuIiEhcZgUWpVKJ0NBQ7Nu3D0OHDgVgGHS7b98+TJkypdpjv/32W2g0GowePbrCc6Vh5cKFCzhw4ADc3d3NKatGRUVFEAQBLVq0gK2tbb2emyyXra0trK2tkZSUBK1WCxsbG7FLIiKiKpjdJTRz5kxERkYiLCwM4eHhWL58OfLy8jBu3DgAwJgxY9CqVSssWbLE5Li1a9di6NChFcJIUVERnn76acTHx+OXX36BTqdDamoqAMDNza1eL9XzygqVx6sqRESWwezAMnz4cNy6dQvz5s1DamoqQkJCsHv3buNA3GvXrlX4EEhMTMSff/6JX3/9tcL5bty4ge3btwMAQkJCTJ47cOAA+vXrZ26JRERE1MSYvQ6LVFU3j7uwsBBXrlxBYGAgL/uTCf63QUQkrgZZh4Wap6tXr0Imk+HEiRNV7rN+/fpq180hIiK6FwwsEjZ27Fjj4GapGz58OM6fP1+rfRluiIjIXE3mbs0kLltb20afgaXT6SCTyThwlogqJwiAoC/zteSBsj+X2wfl9i17LIRy5yr/XNnjUIvXLPm+yufKHldJW6qstfw5qzpvbX8/ZX5+aC6gqv6uyg2lWQYWQRBQUKQT5bVtrRX1Nlvp0KFDmDVrFk6ePAk3NzdERkZi0aJFsLIyvK3btm3DwoULcfHiRdjZ2aF79+746aefYG9vj4MHD+KNN97AmTNnYG1tjc6dO2Pz5s3w9/ev8vUuX76MV199FUePHkVQUBBWrVqFXr16ATBcNZkxYwaysrIAACdPnsSMGTMQGxsLmUyGoKAgrF69Grm5ucYZZaW/h/nz52PBggXIzMzE9OnT8fPPP0Oj0aBv3774+OOPERQUZPIaX375JaKionD+/Hns378fDz30EJKTk03WApoxYwbi4uLwxx9/1MvvmoiqodcDBRlAblrJI73kkXb3a0FmLT4cy33Q3+sHMtW/B15jYGlMBUU6dJq3R5TXPvv2QNgp7/3XfuPGDQwaNAhjx47Fl19+iYSEBLzwwguwsbHBggULkJKSgpEjR+L999/HsGHDkJOTgz/++AOCIKC4uBhDhw7FCy+8gK+//hparRYxMTE1Bqk333wTy5YtQ1BQEN58802MHDkSFy9eNAakskaNGoXu3bvjs88+g0KhwIkTJ2BtbY3evXsbb4CZmJgIAHBwcABg6AK7cOECtm/fDicnJ8yePRuDBg3C2bNnYW1tDcBwd+/33nsPa9asgbu7O/z8/NCmTRt89dVXmDVrFgDDVPlNmzbh/fffv+ffM1GzJQiAJqdM8EgzDSF56abhRBDnH4H1TwbI5ICs9GvJA2V/RjXPlR4rq+a5ssfJKn/O5Njy+4n4mtbirWXWLANLU/Dpp5/Cz88PK1asgEwmw3333YebN29i9uzZxrthFxcX48knnzReNenatSsAICMjA9nZ2XjsscfQtm1bAEDHjh1rfM3XX38dgwcPBgAsXLgQnTt3xsWLF3HfffdV2PfatWuYNWuW8bnSqyQA4OzsDJlMZnJFpDSoHD58GL179wYAbNq0CX5+fvjxxx/xzDPPADCEkU8//RTBwcHGYydMmIDo6GhjYPn5559RWFiIZ599tpa/TaJmpKiwJGxUEUTKhpDiAvPObecOOHgBDp7lvnoBtm6AvKYP3DJfUdOHam2ChRkf5KU/k2Q1y8Bia63A2bcHivba9eHcuXPo1auXyVWRPn36GO+ZFBwcjIceeghdu3bFwIED8fDDD+Ppp5+Gq6sr3NzcMHbsWAwcOBADBgxA//798eyzz8Lb27va1+zWrZvx+9J909PTKw0sM2fOxMSJE/HVV1+hf//+eOaZZ4zhqKr2WFlZISIiwrjN3d0dHTp0wLlz54zblEqlSR2A4crMW2+9hb///hv/+te/sH79ejz77LPV3hGcqEnRFQP5t0vCxq3qg4gm27xzq5wA+xZVBxEHT8PDvgWgsG6Y9hGhmQYWmUxWL90yUqZQKLB371789ddf+PXXX/HJJ5/gzTffxNGjRxEYGIjo6GhMmzYNu3fvxtatW/HWW29h7969+Ne//lXlOUu7ZYC740/0en2l+y5YsADPPfccduzYgV27dmH+/PnYsmULhg0bdk/tsrW1rdB15enpiccffxzR0dEIDAzErl27cPDgwXt6HSLRCYJh3EeV3TBlvubdhlljNhQq07BRVRCx9wSUdg3WRCJzNO1P7SasY8eO+O677yAIgvED/PDhw3B0dISvry8AQ6jo06cP+vTpg3nz5sHf3x8//PCD8S7X3bt3R/fu3TFnzhz06tULmzdvrjawmKt9+/Zo3749Xn31VYwcORLR0dEYNmwYlEoldDrT/u6OHTuiuLgYR48eNXYJ3blzB4mJiejUqVONrzVx4kSMHDkSvr6+aNu2Lfr06VNv7SCqV9q86rthym7TF9X+vDJ5yZWQMqGjwpWRku9tnNn9QRaHgUXisrOzKyzY5u7ujldeeQXLly/H1KlTMWXKFCQmJmL+/PmYOXMm5HI5jh49in379uHhhx+Gp6cnjh49ilu3bqFjx464cuUKPv/8czzxxBPw8fFBYmIiLly4gDFjxtRLzQUFBZg1axaefvppBAYG4vr16zh27BieeuopAEBAQAByc3Oxb98+BAcHw87ODkFBQRgyZAheeOEFrF69Go6OjoiKikKrVq0wZMiQGl9z4MCBcHJywqJFi6q8IzhRgynWAnm3qggiZa+OpAPaXPPObeNSdTdM2W127oC8frqciaSIgUXiDh48iO7du5tsmzBhAtasWYOdO3di1qxZCA4OhpubGyZMmIC33noLAODk5ITff/8dy5cvh1qthr+/Pz744AM8+uijSEtLQ0JCAjZs2IA7d+7A29sbkydPxksvvVQvNSsUCty5cwdjxoxBWloaPDw88OSTT2LhwoUAgN69e+Pll1/G8OHDcefOHeO05ujoaEyfPh2PPfYYtFotHnzwQezcudOkK6oqcrkcY8eOxeLFi+steFEzp9cD+XdKulwqufpRfsquOaztqhkPUqY7xsETsFI1TPuILAzvJURNxoQJE3Dr1i3jzTRrg/9tNDOCAGjU5UJH+SsjpVdEbpk3VVduVX03TNkrI0oHdskQlajtvYR4hYUsXnZ2Nk6dOoXNmzebFVaoCSkqKDcGpOyA1HKzZooLzTu3nUfV3TBlv7dxMUzbJaIGwcBCFm/IkCGIiYnByy+/jAEDBohdDtUnXRGQdQ3IuALkplYxSLWOU3XLd8kYr4yU3ebBqbpEEsHAQhaPU5gtXLEGyEwCMi6XPC7d/T4rufbdMiZTdSvphuFUXSKLxsBCRA2vqBDIvFImlJQ87lwGspNR7RoiVraAWyDg5FN5ICkdnMqpukRNGgMLEdUPbf7dUHKnzFWSjCuA+gaqDSVKB0MocWsDuLUt+VrycGzJIEJEDCxEZAZNjiGAmHTflPyck1L9sSon0yDi1gZwLwkn9i0YSoioWgwsRGSqMLtc102ZqyZ56dUfa+NsuELi3rZiOLFzZyghojpjYCFqjgoy74YRk+6by4ab6FXHzr1cGCkNJ4GAnVvj1E9EzQ4DC1FTJAhAfkbFQa6lM3BqWpnVvkW5sSSBd7+3dWmUJhARlcXA0swEBARgxowZmDFjRoOd4+rVqwgMDMTx48cREhJS59ehGgiC4S695acClz4Ka1ibxKFlyTiScl03roGATdWrTRIRiYGBRaJkNfT1l95/x1zHjh2Dvb19HauqHT8/P6SkpMDDw6PGfRluaiAIhkXSTKYCX7rbnaPNqf54p1blrpCUXDVxDQBUDo3SBCKi+sDAIlEpKXdnXGzduhXz5s1DYmKicZuDw90PG0EQoNPpYGVV89vZokWL+i20EgqFAi1btmzw1ymvqKioVjdKlBy93jDDpkL3TUkoKcqr5mAZ4OxbcYBraUixtm20ZhARNaTmeeMLQQC0eeI8anmvyZYtWxofzs7OkMlkxp8TEhLg6OiIXbt2ITQ0FCqVCn/++ScuXbqEIUOGwMvLCw4ODujZsyd+++03k/MGBARg+fLlxp9lMhnWrFmDYcOGwc7ODkFBQbW6H09+fj7Gjx8PR0dHtG7dGp9//rnxuatXr0Imk+HEiRMAgMzMTIwaNQotWrSAra0tgoKCEB0dDQAIDAwEAHTv3h0ymQz9+vUDAOj1erz99tvw9fWFSqVCSEgIdu/eXeE1tm7dir59+8LGxgaff/45nJycsG3bNpNaf/zxR9jb2yMnp4arEQ1Jrzes2nr5EBC7Dvj1LWDLKODTXsBiH+CjTsCGx4CfpwGHlwPntgNppw1hRSYHXPyBNv8GwiYAAxcDI7cAk2OAN1OBV08DkduBx5cDfaYBHR8DvDoxrBBRk9I8r7AU5Rs+JMTwn5uAsn66ZKKiorBs2TK0adMGrq6uSE5OxqBBg/Df//4XKpUKX375JR5//HEkJiaidevWVZ5n4cKFeP/997F06VJ88sknGDVqFJKSkuDmVvWMjw8++ADvvPMO/vOf/2Dbtm2YNGkS+vbtiw4dOlTYd+7cuTh79ix27doFDw8PXLx4EQUFBQCAmJgYhIeH47fffkPnzp2hVCoBAP/73//wwQcfYPXq1ejevTvWrVuHJ554AmfOnEFQUJDJ7+CDDz5A9+7dYWNjg5MnTyI6OhpPP/20cZ/Snx0dHc3+HZtFrzOs2lr26khp903mVUCnqfpYmQJw9a/kKklbwKU1YKVs2NqJiCSueQaWJuLtt982udmfm5sbgoODjT+/8847+OGHH7B9+3ZMmTKlyvOMHTsWI0eOBAAsXrwYH3/8MWJiYvDII49UecygQYPwyiuvAABmz56Njz76CAcOHKg0sFy7dg3du3dHWFgYAMNVnlKlXVTu7u4m3UjLli3D7NmzMWLECADAe++9hwMHDmD58uVYuXKlcb8ZM2bgySefNP48ceJE9O7dGykpKfD29kZ6ejp27txZ4UpTnemKgaykcounXb4bSvRFVR8rty4TSsrNwHFpzZvsERFVo3kGFms7w5UOsV67npQGgFK5ublYsGABduzYgZSUFBQXF6OgoADXrl2r9jzdunUzfm9vbw8nJyekp1e/QFjZY0q7q6o6ZtKkSXjqqacQHx+Phx9+GEOHDkXv3r2rPLdarcbNmzfRp08fk+19+vTByZMnTbaV/x2Eh4ejc+fO2LBhA6KiorBx40b4+/vjwQcfrLY9Joq1JXcIruxmfNcAfXHVxyqUhlk2xpVcy1wtcfIFFM3zT46I6F41z/97ymT11i0jpvKzfV5//XXs3bsXy5YtQ7t27WBra4unn34aWq222vOUH6gqk8mg1+vr7ZhHH30USUlJ2LlzJ/bu3YuHHnoIkydPxrJly6p9jdqobMbTxIkTsXLlSkRFRSE6Ohrjxo2rOOtKrwd0WsNS85oc4OB7wK0Thi6c7GRAqKb9VjaVr0/i1tZwgz654p7bRUREpppnYGmiDh8+jLFjx2LYsGEADFdcrl69Km5RJVq0aIHIyEhERkbigQcewKxZs7Bs2TLjmBWdTmfc18nJCT4+Pjh8+DD69u1r3H748GGEh4fX+FqjR4/GG2+8gY8//hhnz55F5IinDFODizWGh05reABAsWBYRO30t0Bu8t2TWNuVCSTlb8bnDcib53h1IiKxMLA0IUFBQfj+++/x+OOPQyaTYe7cuTVeKWkM8+bNQ2hoKDp37gyNRoNffvkFHTt2BAB4enrC1tYWu3fvhq+vL2xsbODs7IxZs2Zh/vz5aNu2LUJCQhAdHY0TJ05g06ZNNb6eq6srnhw2DLNmzcLD/XrD17YAUBdU3FEmN4wbsbYDeowFXLzu3gPHwYv3vSEikhAGlibkww8/xPjx49G7d294eHhg9uzZUKvVYpcFpVKJOXPm4OrVq7C1tcUDDzyALVu2AACsrKzw8ccf4+2338a8efPwwAMP4ODBg5g2bRqys7Px2muvIT09HZ06dcL27dtNZghVSlcM5N/GhCf/D5u//hrjn33cEExUToCVyvBQlHyVWwEaDaCWAV2mADY2jfDbICKiupAJQi0XBpE4tVoNZ2dnZGdnw8nJdFnxwsJCXLlyBYGBgbDhh1LTVFwI5N4y3D8Heny17Re8uuBD3Lz4D5Qu3oZwUgn+t0FEJK7qPr/L4hUWslylCwDmpgMaw31z8gsKkHInF++u2oSXXp4EpZufyEUSEVF94MhBsjxCyUDZ2+eBOxeMYQUqJ7wf/TPu6zMYLb1bYc5//iNunUREVG94hYUsh14H5N8B8m7dneUDGWDnBth7AtY2WLBoCRYsWiJqmUREVP8YWEj6irVA/i0g7w4glEx/llsBdh6AvQdXiCUiagaaVWBpIuOLmw9tPpCXDhRkASh57xQqwMETsHWrl7VQ+N8EEZFlaBaBRaEwrDyq1Wpha8s72EqaIBhWns1NA7S5d7crHQxBReVUr+uj5OfnA6i4ci8REUlLswgsVlZWsLOzw61bt2BtbQ05VymVHr0eKMwGCjJM72qsdAbsXAFlyT2YNNXc8dgMgiAgPz8f6enpcHFxMYZaIiKSpmYRWGQyGby9vXHlyhUkJSWJXQ6VpdcD2hxAk3t3fIpMbrjXk8oRkGuBzLQGe3kXFxeTu0QTEZE0NYvAAhhWWw0KCqrxRoDUSDKvASc2Awm/ALpCwzZ7LyB4BNBpqCGsNDBra2teWSEishDNJrAAgFwu52qmYhIE4NrfwJEVQMIOGAfStuwG9J4GdB7KGT9ERFSpZhVYSCS6YiDhZ+CvT4AbcXe3Bw0Eek8BAh7gjQaJiKhaDCzUcDQ5wPGNwN+fAlnXDNsUKiB4ONBrCtCig7j1ERGRxWBgofqnvgkcXQ3ERt9dNt/WDQh/Aeg50TA9mYiIyAwMLFR/Uk8Bf60ATm8D9MWGbW5tgV6TgeCRd6cmExERmYmBhe6NIAAX9wF/fQxcOXR3u38fQ7dP+0fqZUVaIiJq3hhYqG6KNcA/3wBHVgK3zhm2yRRApyGGgbStQsWtj4iImhQGFjJPfgYQuxaI+cKwfD5gWDa/RyTwr5cBl9bi1kdERE0SAwvVzp1Lhtk+xzcBxQWGbY4+hpDSIxKwdRG1PCIiatoYWKhqggAkHzWsn1JhobepQOdhXOiNiIgaBQMLVWRc6G0FcCP27vaghw0DaQMf5EJvRETUqBhY6C5NbpmF3kpuEqlQAt1KFnrzvE/c+oiIqNliYKG7C73FRQOFZRZ66znRsNgbF3ojIiKRMbA0Z6mnDTciPLUN0BcZtnGhNyIikiAGluamdKG3I58Alw/e3d66t2H9lPaPcqE3IiKSHAaW5qJYA5z61rDQW/pZwzaZ3LDQW6+pgC8XeiMiIuliYGnq8jOA2HVAzOflFnobA0S8DLj6i1sfERFRLTCwNFUZl4EjnwInNgFF+YZtXOiNiIgsFANLU3PtqOFGhCYLvXU1dPt0HgZYKUUtj4iIqC4YWJoCvQ4497Nhxs/1Y3e3txtgWJGWC70REZGFY2CxZJpcQ5fPkZVc6I2IiJo0BhZLpE4BYlYbBtMaF3pzNSz01vMFwNFL3PqIiIjqGQOLJUk9bbiacurbcgu9vQIEP8eF3oiIqMmq0wphK1euREBAAGxsbBAREYGYmJgq9+3Xrx9kMlmFx+DBg437CIKAefPmwdvbG7a2tujfvz8uXLhQl9KaHkEALv4GfDkUWNUHOLnZEFZa9wKGbwKmHDNcWWFYISKiJszswLJ161bMnDkT8+fPR3x8PIKDgzFw4ECkp6dXuv/333+PlJQU4+P06dNQKBR45plnjPu8//77+Pjjj7Fq1SocPXoU9vb2GDhwIAoLC+veMktXrAGObwI+6wNsfAq4fMCw0FvnYcDE/cD43UDHxwC5QuxKiYiIGpxMEATBnAMiIiLQs2dPrFixAgCg1+vh5+eHqVOnIioqqsbjly9fjnnz5iElJQX29vYQBAE+Pj547bXX8PrrrwMAsrOz4eXlhfXr12PEiBG1qkutVsPZ2RnZ2dlwcnIyp0nSUtlCb9b2hoXe/vUy4BoganlERET1qbaf32aNYdFqtYiLi8OcOXOM2+RyOfr3748jR47U6hxr167FiBEjYG9vDwC4cuUKUlNT0b9/f+M+zs7OiIiIwJEjR6oMLBqNBhqNxvizWq02pynSk3EZ+Psz4PhG04XeIl4CQsdyoTciImrWzAost2/fhk6ng5eX6SwULy8vJCQk1Hh8TEwMTp8+jbVr1xq3paamGs9R/pylz1VmyZIlWLhwoTnlS9O1o4YbEZ77BcaF3ry6Gm5E2PlJLvRGRESERp4ltHbtWnTt2hXh4eH3fK45c+Zg5syZxp/VajX8/Pzu+byNQq8DEn4B/loBXC8zYLndAENQCezLhd6IiIjKMCuweHh4QKFQIC0tzWR7WloaWrZsWe2xeXl52LJlC95++22T7aXHpaWlwdvb2+ScISEhVZ5PpVJBpVKZU774tHmGgbR/rwQyrxq2KZRAt2dLFnrrKGp5REREUmXWLCGlUonQ0FDs27fPuE2v12Pfvn3o1atXtcd+++230Gg0GD16tMn2wMBAtGzZ0uScarUaR48erfGcFiMnFfhtIfBhJ2DXLENYsXUFHpwFzDgNDFnJsEJERFQNs7uEZs6cicjISISFhSE8PBzLly9HXl4exo0bBwAYM2YMWrVqhSVLlpgct3btWgwdOhTu7u4m22UyGWbMmIFFixYhKCgIgYGBmDt3Lnx8fDB06NC6t0wK0s4Yun1MFnprA/zrFSDkOUBpL259REREFsLswDJ8+HDcunUL8+bNQ2pqKkJCQrB7927joNlr165BLje9cJOYmIg///wTv/76a6XnfOONN5CXl4cXX3wRWVlZuP/++7F7927Y2NjUoUkiEwTg0n7DjQgv7b+7vXUvQ7dPh0e5dgoREZGZzF6HRapEX4elWAuc3ma4opJ+xrBNJgc6PmG4Y7JvWOPXREREJHENsg4LVSI/A4iLBo5+DuSWTMPmQm9ERET1ioGlrjKulCz09lWZhd68yyz05ipqeURERE0JA4u5kmOAvz4xrKMi6A3buNAbERFRg2JgqQ29DkjYYQgqJgu99TcMpG3Tjwu9ERERNSAGluoUFQDxXwF/fwpkXjFs40JvREREjY6BpTp6HXDgv0BhlmFMStgEIPxFwNGrxkOJiIio/jCwVEflAPz7P4bpyVzojYiISDQMLDWJeEnsCoiIiJo9s+4lRERERCQGBhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSvDoFlpUrVyIgIAA2NjaIiIhATExMtftnZWVh8uTJ8Pb2hkqlQvv27bFz507j8zqdDnPnzkVgYCBsbW3Rtm1bvPPOOxAEoS7lERERURNjZe4BW7duxcyZM7Fq1SpERERg+fLlGDhwIBITE+Hp6Vlhf61WiwEDBsDT0xPbtm1Dq1atkJSUBBcXF+M+7733Hj777DNs2LABnTt3RmxsLMaNGwdnZ2dMmzbtnhpIRERElk8mmHkZIyIiAj179sSKFSsAAHq9Hn5+fpg6dSqioqIq7L9q1SosXboUCQkJsLa2rvScjz32GLy8vLB27Vrjtqeeegq2trbYuHFjrepSq9VwdnZGdnY2nJyczGkSERERiaS2n99mdQlptVrExcWhf//+d08gl6N///44cuRIpcds374dvXr1wuTJk+Hl5YUuXbpg8eLF0Ol0xn169+6Nffv24fz58wCAkydP4s8//8Sjjz5aZS0ajQZqtdrkQURERE2TWV1Ct2/fhk6ng5eXl8l2Ly8vJCQkVHrM5cuXsX//fowaNQo7d+7ExYsX8corr6CoqAjz588HAERFRUGtVuO+++6DQqGATqfDf//7X4waNarKWpYsWYKFCxeaUz4RERFZqAafJaTX6+Hp6YnPP/8coaGhGD58ON58802sWrXKuM8333yDTZs2YfPmzYiPj8eGDRuwbNkybNiwocrzzpkzB9nZ2cZHcnJyQzeFiIiIRGLWFRYPDw8oFAqkpaWZbE9LS0PLli0rPcbb2xvW1tZQKBTGbR07dkRqaiq0Wi2USiVmzZqFqKgojBgxAgDQtWtXJCUlYcmSJYiMjKz0vCqVCiqVypzyiYiIyEKZdYVFqVQiNDQU+/btM27T6/XYt28fevXqVekxffr0wcWLF6HX643bzp8/D29vbyiVSgBAfn4+5HLTUhQKhckxRERE1HyZ3SU0c+ZMfPHFF9iwYQPOnTuHSZMmIS8vD+PGjQMAjBkzBnPmzDHuP2nSJGRkZGD69Ok4f/48duzYgcWLF2Py5MnGfR5//HH897//xY4dO3D16lX88MMP+PDDDzFs2LB6aCIRERFZOrPXYRk+fDhu3bqFefPmITU1FSEhIdi9e7dxIO61a9dMrpb4+flhz549ePXVV9GtWze0atUK06dPx+zZs437fPLJJ5g7dy5eeeUVpKenw8fHBy+99BLmzZtXD00kIiIiS2f2OixSxXVYiIiILE+DrMNCREREJAYGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BpZa0OsFsUsgIiJq1hhYarDmj8uYsOEYtMV6sUshIiJqthhYqpGSXYAPfj2PA4m3MGPrcRTrGFqIiIjEwMBSDW9nW6x+PhRKhRw7T6Ui6vtT7B4iIiISAQNLDR5s3wIfj+wOhVyGbXHX8fYvZyEIDC1ERESNiYGlFh7p0hJLn+4GAFj/11V8uPe8yBURERE1LwwstfRkD1+8M6QzAOCT/Rex6tAlkSsiIiJqPhhYzPB8rwDMfuQ+AMC7uxKw8e8kkSsiIiJqHhhYzDSpX1tM/ndbAMDcn07jh+PXRa6IiIio6WNgqYPXH+6AyF7+EATg9W//wZ4zqWKXRERE1KQxsNSBTCbD/Mc746kevtDpBUzdfBx/XLgldllERERNFgNLHcnlMrz3VFc82qUltDo9XvwyDrFXM8Qui4iIqEliYLkHVgo5lo8IQd/2LVBQpMO46GM4fSNb7LKIiIiaHAaWe6SyUmDV6FCEB7ghR1OMMeticDE9R+yyiIiImhQGlnpgq1Rg7dgwdPN1RkaeFqPWHEVyRr7YZRERETUZDCz1xNHGGhvGhaO9lwPS1Bo8t+ZvpGYXil0WERFRk8DAUo9c7ZXYOCEC/u52SM4owOi1R5GRpxW7LCIiIovHwFLPPJ1ssHFCBFo62eBiei7GrDsKdWGR2GURERFZNAaWBuDnZoeNEyPgbq/E6RtqjI8+hnxtsdhlERERWSwGlgbSztMBX04Ih6ONFWKTMvHSV3HQFOvELouIiMgiMbA0oM4+zlg/Lhx2SgX+uHAb074+jmKdXuyyiIiILA4DSwML9XfFF2PCoFTIsedMGt7Y9g/0ekHssoiIiCwKA0sj6NPOAytH9YBCLsP3x29gwc9nIAgMLURERLXFwNJIBnTywofPBkMmA748koSlexLFLomIiMhiMLA0oiEhrfDfoV0BAJ8evISVBy6KXBEREZFlYGBpZM9FtMabgzoCAJbuScSXR66KWxAREZEFYGARwQsPtsG0/2sHAJj30xlsi7suckVERETSxsAiklcHtMe4PgEAgDe2ncSuUyniFkRERCRhDCwikclkmDu4E54N84VeAKZtOY5D52+JXRYREZEkMbCISC6XYcmT3TC4qzeKdAJe+ioWMVcyxC6LiIhIchhYRKaQy/DR8BD8u0MLFBbpMX79MfxzPUvssoiIiCSFgUUClFZyfDY6FBGBbsjVFCNyXQzOp+WIXRYREZFkMLBIhI21AmvH9kSwnwsy84swes1RJN3JE7ssIiIiSWBgkRAHlRU2jOuJDl6OSM/RYNSao0jJLhC7LCIiItExsEiMi50SX00MR4C7Ha5nFmD0mqO4nasRuywiIiJRMbBIkKejDTZOjICPsw0u3crDmLUxyC4oErssIiIi0TCwSJSvqx02ToyAh4MSZ1PUGBcdgzxNsdhlERERiYKBRcLatHDAVxMi4GRjhfhrWXjxq1gUFunELouIiKjRMbBIXEdvJ2wYHw47pQKHL97B1K+Po0inF7ssIiKiRsXAYgG6t3bFmsgwKK3k2Hs2Da9/exJ6vSB2WURERI2GgcVC9G7rgc9G9YCVXIafTtzEWz+dhiAwtBARUfPAwGJBHurohY+Gh0AmAzYfvYYluxIYWoiIqFlgYLEwjwf74N0nuwIAPv/9MlbsvyhyRURERA2PgcUCDe/ZGnMf6wQA+GDveaz784rIFRERETUsBhYLNeH+QLzavz0A4O1fzuKbY8kiV0RERNRwGFgs2LSH2uGFBwIBAFHf/4Nf/rkpckVEREQNg4HFgslkMvxnUEeMDPeDXgBmbDmBAwnpYpdFRERU7xhYLJxMJsOioV3xRLAPivUCXt4Yh78v3xG7LCIionpVp8CycuVKBAQEwMbGBhEREYiJial2/6ysLEyePBne3t5QqVRo3749du7cabLPjRs3MHr0aLi7u8PW1hZdu3ZFbGxsXcprdhRyGT54Nhj9O3pCU6zHhPXHcCI5S+yyiIiI6o3ZgWXr1q2YOXMm5s+fj/j4eAQHB2PgwIFIT6+8K0Kr1WLAgAG4evUqtm3bhsTERHzxxRdo1aqVcZ/MzEz06dMH1tbW2LVrF86ePYsPPvgArq6udW9ZM2OtkGPFcz3Qu6078rQ6RK6LQUKqWuyyiIiI6oVMMHPlsYiICPTs2RMrVqwAAOj1evj5+WHq1KmIioqqsP+qVauwdOlSJCQkwNrautJzRkVF4fDhw/jjjz/q0AQDtVoNZ2dnZGdnw8nJqc7nsXR5mmKMXnsUx69lwcNBhW9f7oVAD3uxyyIiIqpUbT+/zbrCotVqERcXh/79+989gVyO/v3748iRI5Ues337dvTq1QuTJ0+Gl5cXunTpgsWLF0On05nsExYWhmeeeQaenp7o3r07vvjii2pr0Wg0UKvVJg8C7FVWWD82HB29nXA7V4PRa47iZlaB2GURERHdE7MCy+3bt6HT6eDl5WWy3cvLC6mpqZUec/nyZWzbtg06nQ47d+7E3Llz8cEHH2DRokUm+3z22WcICgrCnj17MGnSJEybNg0bNmyospYlS5bA2dnZ+PDz8zOnKU2as501vhwfjjYe9riRVYDRa47iVo5G7LKIiIjqrMFnCen1enh6euLzzz9HaGgohg8fjjfffBOrVq0y2adHjx5YvHgxunfvjhdffBEvvPCCyT7lzZkzB9nZ2cZHcjIXTiurhaMKGydGoJWLLS7fzsPza48iO79I7LKIiIjqxKzA4uHhAYVCgbS0NJPtaWlpaNmyZaXHeHt7o3379lAoFMZtHTt2RGpqKrRarXGfTp06mRzXsWNHXLt2rcpaVCoVnJycTB5kysfFFpsmRqCFowoJqTmIjI5BrqZY7LKIiIjMZlZgUSqVCA0Nxb59+4zb9Ho99u3bh169elV6TJ8+fXDx4kXo9XrjtvPnz8Pb2xtKpdK4T2Jioslx58+fh7+/vznlUSUCPOyxcUIEXOyscSI5Cy9siEVhka7mA4mIiCTE7C6hmTNn4osvvsCGDRtw7tw5TJo0CXl5eRg3bhwAYMyYMZgzZ45x/0mTJiEjIwPTp0/H+fPnsWPHDixevBiTJ0827vPqq6/i77//xuLFi3Hx4kVs3rwZn3/+uck+VHcdWjpiw7hw2CsVOHL5DiZvikeRTl/zgURERBJhdmAZPnw4li1bhnnz5iEkJAQnTpzA7t27jQNxr127hpSUFOP+fn5+2LNnD44dO4Zu3bph2rRpmD59uskU6J49e+KHH37A119/jS5duuCdd97B8uXLMWrUqHpoIgFAsJ8L1o7tCZWVHPsS0jHzm5PQ6c2a0U5ERCQas9dhkSquw1I7BxLT8eKXsSjSCRjR0w9LnuwKmUwmdllERNRMNcg6LGT5/t3BE8uHd4dcBmw5loxFO86hiWRWIiJqwhhYmqHB3bzx7lPdAABr/7yC/+27IHJFRERE1WNgaaaeDfPD/McNU8mX/3YBa/64LHJFREREVWNgacbG9QnE6w+3BwAs2nEOX8dUve4NERGRmBhYmrnJ/26Hl/q2AQD854dT2H7ypsgVERERVcTA0szJZDJEPXIfRkW0hiAAM7eewG9n02o+kIiIqBExsBBkMhneGdIFw7q3QrFewCub4/HXxdtil0VERGTEwEIAALlchqVPd8PDnbygLdZj4pexiL+WKXZZREREABhYqAwrhRyfPNcdDwR5IF+rw9h1MTh7Uy12WURERAwsZEplpcDq50MR6u8KdWExxqw7iku3csUui4iImjkGFqrATmmFdWN7orOPE27najF6zVFcz8wXuywiImrGGFioUs621vhyfDjatrBHSnYhRq05inR1odhlERFRM8XAQlVyd1Bh08R/wc/NFkl38vH82hhk5WvFLouIiJohBhaqVktnG2ya8C94OqqQmJaDyHUxyNUUi10WERE1MwwsVKPW7nbYNDECrnbWOHk9GxPWH0NhkU7ssoiIqBlhYKFaCfJyxJfjI+CossLRKxmYtDEO2mK92GUREVEzwcBCtdbV1xnrxvWEjbUcBxJv4dWtJ6DTC2KXRUREzQADC5mlZ4AbPn8+DEqFHDtOpSDqu3+gZ2ghIqIGxsBCZnuwfQt8PDIEchnwbdx1vP3LWQgCQwsRETUcBhaqk0e6eGPp08EAgPV/XcVHe8+LXBERETVlDCxUZ0+F+uLtIZ0BAB/vv4jVhy6JXBERETVVDCx0T8b0CsAbj3QAACzZlYBNR5NEroiIiJoiBha6Z6/0a4dX+rUFALz142n8ePyGyBUREVFTw8BC9WLWwA4Y08sfggC89u1J/HomVeySiIioCWFgoXohk8mw4PHOeLJHK+j0AqZsPo4/L9wWuywiImoiGFio3sjlMrz/VDc80rkltDo9XvgyFnFJGWKXRURETQADC9UrK4Uc/xsZggfbt0BBkQ5jo4/h9I1sscsiIiILx8BC9U5lpcDq0aEID3BDTmExxqyLwcX0HLHLIiIiC8bAQg3CVqnAmrFh6NrKGRl5WoxeE4PkjHyxyyIiIgvFwEINxsnGGhvGhyPI0wGp6kKMWnMUaepCscsiIiILxMBCDcrNXomNEyPQ2s0O1zLyMXrNUWTkacUui4iILAwDCzU4LycbbJoYgZZONriQnovIdTFQFxaJXRYREVkQBhZqFH5udtg4MQJu9kqcupGNCeuPoUCrE7ssIiKyEAws1GjaeTrgy/HhcLSxwrGrmXjxq1hoihlaiIioZgws1Ki6tHLG+nE9YWutwB8XbmP61ydQrNOLXRYREUkcAws1ulB/N3wxJgxKhRy7z6Tije/+gV4viF0WERFJGAMLieL+IA+seK47FHIZvo+/gQU/n4EgMLQQEVHlGFhINA93bokPngmGTAZ8eSQJS/ckil0SERFJFAMLiWpo91ZYNLQLAODTg5fw6cGLIldERERSxMBCohsV4Y//DLoPAPD+7kR8deSquAUREZHkMLCQJLz4YFtM+792AIC5P53Bd3HXRa6IiIikhIGFJOPVAe0xrk8AAGDWtpPYfTpF3IKIiEgyGFhIMmQyGeYO7oRnw3yhF4CpXx/HofO3xC6LiIgkgIGFJEUul2HJk90wuKs3inQCXvoqFseuZohdFhERiYyBhSRHIZfho+Eh6NehBQqL9BgffQynrmeLXRYREYmIgYUkSWklx6rRoYgIdEOOphhj1h3FhbQcscsiIiKRMLCQZNlYK7AmMgzBvs7IzC/CqDVHce1OvthlERGRCBhYSNIcbayxflw4Ong5Ij1Hg+fW/I3U7EKxyyIiokbGwEKS52qvxFcTwhHgbofrmQUYteZv3MnViF0WERE1IgYWsgieTjbYODEC3s42uHQrD2PWxSC7oEjssoiIqJEwsJDF8HW1w6aJEfBwUOLMTTXGrz+GfG2x2GUREVEjYGAhi9KmhQO+HB8BJxsrxCVl4sUv41BYpBO7LCIiamAMLGRxOvk4Yf34cNgpFfjz4m1M/fo4inR6scsiIqIGxMBCFqlHa1esGRMGpZUce8+mYda3J6HXC2KXRUREDYSBhSxW73Ye+PS5HrCSy/DjiZuY+9NpCAJDCxFRU8TAQhatfycvfDg8BDIZsOnoNby7K4GhhYioCWJgIYv3RLAPlgzrCgBY/ftlrDxwUeSKiIiovjGwUJMwIrw13hrcEQCw7NfziD58ReSKiIioPjGwUJMx8YE2mNE/CACw8Oez+CY2WeSKiIiovjCwUJMy/aEgTLw/EAAQ9d0/2PFPisgVERFRfWBgoSZFJpPhzcEdMaKnH/QCMGPrcRxISBe7LCIiukcMLNTkyGQy/HdYVzwe7IMinYCXN8bhl39uIlfDZfyJiCyVldgFEDUEhVyGD58NRoG2GL+dS8eUzcchlwEdvZ0Q5u+K0AA3hPm7wsfFVuxSiYioFmRCE1m0Qq1Ww9nZGdnZ2XBychK7HJKIwiId3t2VgN/OpeF6ZkGF51u52CLU3xVhAa4I9XfFfS2doJDLRKiUiKh5qu3nNwMLNRtp6kLEXs3EsasZiEvKxNkUNXTllvN3UFmhe2sXhPm7ISzAFSF+LrBX8UIkEVFDqe3nd53GsKxcuRIBAQGwsbFBREQEYmJiqt0/KysLkydPhre3N1QqFdq3b4+dO3dWuu+7774LmUyGGTNm1KU0oip5OdlgcDdvLHiiM36eej/+mf8wNk+MwMwB7fFAkAccVFbI1RTjjwu38dFv5zFqzVF0W/grHvvkDyzYfga//HMTqdmFYjeDiKhZMvufjlu3bsXMmTOxatUqREREYPny5Rg4cCASExPh6elZYX+tVosBAwbA09MT27ZtQ6tWrZCUlAQXF5cK+x47dgyrV69Gt27d6tQYInPYq6zQu50HerfzAADo9AISU3MQl5SB2KRMxF7NxI2sApy+ocbpG2qs/+sqAEM3UliAK8L8XREW4Ib2Xo7sRiIiamBmdwlFRESgZ8+eWLFiBQBAr9fDz88PU6dORVRUVIX9V61ahaVLlyIhIQHW1tZVnjc3Nxc9evTAp59+ikWLFiEkJATLly+vdV3sEqKGkJJdgNirmYhLykRsUgbO3lSj/E2hHVVW6O5fEmD8XRHS2gV2SnYjERHVRoOMYdFqtbCzs8O2bdswdOhQ4/bIyEhkZWXhp59+qnDMoEGD4ObmBjs7O/z0009o0aIFnnvuOcyePRsKhcLkHG5ubvjoo4/Qr1+/GgOLRqOBRqMxabCfnx8DCzWoXE0xTlzLQmySYRxMfFIm8rQ6k30Uchk6+zgZBvOWjIXxcrIRqWIiImmrbWAx65+Bt2/fhk6ng5eXl8l2Ly8vJCQkVHrM5cuXsX//fowaNQo7d+7ExYsX8corr6CoqAjz588HAGzZsgXx8fE4duxYrWtZsmQJFi5caE75RPfMQWWF+4M8cH/Q3W6khFQ1Yq9mIjYpE3FXM3AzuxD/XM/GP9ezEX34KgDA19XW2IUUFuCK9p6OkLMbiYio1hr8urVer4enpyc+//xzKBQKhIaG4saNG1i6dCnmz5+P5ORkTJ8+HXv37oWNTe3/FTpnzhzMnDnT+HPpFRaixmS4muKMzj7OiOwdAAC4mVVQMgYmA7FXM5GQqsb1zAJczyzAjyduAgAcbazQo/XdcTAhfi6wVSqqeSUioubNrMDi4eEBhUKBtLQ0k+1paWlo2bJlpcd4e3vD2trapPunY8eOSE1NhVarRVxcHNLT09GjRw/j8zqdDr///jtWrFgBjUZjcmwplUoFlUplTvlEjcLHxRZPuNjiiWAfAEBOYRFOJGcZx8LEX8tETmExDp2/hUPnbwEArIzdSG7GAb2e7EYiIjIyK7AolUqEhoZi3759xjEser0e+/btw5QpUyo9pk+fPti8eTP0ej3kcsMs6vPnz8Pb2xtKpRIPPfQQTp06ZXLMuHHjcN9991UY50JkiRxtrPFAUAs8ENQCAFCs0yMhNcdwBaZkNlKquhAnr2fj5PVsrDt8BQDg52aLnv5uCA0wjIUJ8nRgNxIRNVtmzxLaunUrIiMjsXr1aoSHh2P58uX45ptvkJCQAC8vL4wZMwatWrXCkiVLAADJycno3LkzIiMjMXXqVFy4cAHjx4/HtGnT8Oabb1b6GrUZdFseZwmRpRIEATeyCgwzkUrGwiSkqlH+L9PJxgo9/F3RM8ANof6uCPZlNxIRWb4GGXQLAMOHD8etW7cwb948pKamIiQkBLt37zYOxL127ZrxSgoA+Pn5Yc+ePXj11VfRrVs3tGrVCtOnT8fs2bPr0Cyipkcmk8HX1Q6+rnYYEtIKAKAuLMLxa1mIK7kKcyI5C+rCYhxMvIWDiWW6kVo5I8zfFT0DXBHq74YWjuwmJaKmiUvzE1mAYp0e51JyEJuUUXIVJgNpak2F/fzd7UymU7drwW4kIpI23kuIqAkTBAHXMwuMC9rFXs1EYlpOhW4kZ1trhPq7loQYVwT7ucDGmt1IRCQdDCxEzUx2QRGOX8s0joU5npyJwiK9yT7WCsM07NIupLAAV3g4sBuJiMTDwELUzBXp9Dh7U21Y0K7kKkx6TsVupAB3O4T6u6FngCvCAlzRtoUDZDJ2IxFR42BgISITpd1Ix0oG8sZdzcT59IrdSC521ght7WqcTt3N15ndSETUYBhYiKhG2QVFiL92d1Xek9ezKnQjKRVydGnlhLCS6dRh/q5wZzcSEdUTBhYiMpu2WI+zKWpjgIlNysTt3IrdSG087A3hpWQsTNsW9uxGIqI6YWAhonsmCAKuZeTfvbljUgbOp+VW2M/VzrokwLghzN8VXVqxG4mIaoeBhYgaRFa+tqQbyRBiTiZnQVNcsRupq69hUbvSIONmrxSpYiKSMgYWImoU2mI9Tt/MRlzJgnZxSZm4nautsF+bFvaGu1OX3B+pjQe7kYiIgUXscoiaLUEQkHQnH8euZpQsbJeJi+kVu5Hc7ZXoUTKINyzA0I2ksmI3ElFzw8BCRJKRmVfSjZRkmJF08no2tOW7kazkCPZ1NixoV9KV5MpuJKImj4GFiCRLU6zD6Rtq44J2sUmZyMir2I3UtoW9sQupnacDfF1t0cJBxa4koiaEgYWILIYgCLhyO8+4oF1sUgYu3cqrdF+VlRy+rrbwc7ODn6ud8XtfV1v4udrBxc6agYbIgjCwEJFFy8jTGm/ueDI5C9fu5CNVXQh9Df/HclBZwdfVtuRhZxJmfN1s4WRj3TgNIKJaYWAhoiZHW6xHSnYBrmcWIDkj3/A1M9/4fWX3SirP2dbaGGD83EpDjeGrr6st7JRWjdASIipV289v/mUSkcVQWsnh724Pf3f7Sp8vLNLhRpZpmLmeUYDrmflIzixARp4W2QVFyC4owpmb6krP4W6vhK+bHfxcTcOMn6stWrnaciYTkUgYWIioybCxVqBtCwe0beFQ6fN5mmJczywJMBmGEGP43vBVXViMO3la3MnT4mRyVqXn8HJSVTp2xs/NDi2dbWCtkDdgC4maL3YJERGVyC4oMgkw5bue8rW6ao+XywBvZ9sKYab0Zy8nGyjkHBBMVBbHsBAR1SNBEJCZX1Tp2JnkknBTfm2Z8qwVMvi42Fa4QlPa9cQp29QccQwLEVE9kslkcLNXws1eiWA/lwrP6/UCbudqynQzmYaZG5kFKNIZVgFOupNf6WuUTtk2HTtz93tXTtmmZoyBhYioHsjlMng62cDTyQah/q4VntfpBaSpCysdO3M9swAp2QXQFOtx6VZelWvQ2CsVFWY1Gbue3Ow4ZZuaNAYWIqJGoJAbuoN8XGwRUcnzRTo9UrIKS2Y03Q0zpeEmTa1BnlaHxLQcJKblVPoaTjZWFQYClw01nLJNloz/9RIRSYC1Qo7W7nZo7W5X6fOFRTrczCpAcrmBwNczC3A9Ix938rRQFxbjzE119VO2XW3hW0moaeViCxtrTtkm6WJgISKyADbWCrRp4YA2VUzZztcWm85qKhdqsguK7k7Zvp5d6Tk8HVXlrtDcHUfj7cIp2yQuBhYioibATmmF9l6OaO/lWOnz6sKiCmGm7NTtPK0O6TkapOdoEJeUWeH40inbrSqEGcMVm5acsk0NjNOaiYiaudIp29dNxs6YhhtNDVO2rUrG6Pi5VZy27edqBw8HFeQMNFQJTmsmIqJaKTtlu5uvS4XnBUHArVyNyaymsuHmRpZhyva1jHxcy8gHcKfCOZQKufE13B2UcLdXwt1BZfi5zPceDoZ9HFRWnMJNJhhYiIioWjKZDJ6ONvB0rH7KdsWxM4ZQk5JdAK1Oj1R1IVLVhbV6TaVCDneH0oCjgnu5sONmrzIJPvZKBQNOE8fAQkRE96TslO3wQLcKzxfp9EjP0SAjV4vbeYavGXl3vy8dDJyRp8GdXC3ytTpodXqkZBciJbuWAcdKXhJeDGHGoyTguDko4WGvKhN2VHBzUDLgWCAGFiIialDWCjlauRimTtdGgVaHO3kaZJQEmTu5d8OMIdhocSdXY3yuoEgHbbF5AUdVEnDcSkJM2bBz93slPEq6quwYcETHwEJERJJiq1TAV2kHX9fK16QprzTg3Cm5cnOnJNBk5GlxuyTslH5/J0+DwiI9NMV63MwuxE0zAk5peDF2VVUYe3M37HCRvvrH3ygREVk0cwNOvra4TLgpf+VGe/fqTrmAcyOrADeyCmr1GjbWcsOVG2O4UZULOiXdUww4tcbfEBERNSt2SivYuRluY1AbpQHnTpmuqIwK35eMy8nVQFOsR2GReQHH1lphMkvKzV5l/L7ioGMVbJXNb1ViBhYiIqJqmBNwBEFAvlZnDC93r9SYjsO5k1c6CFkLbbEeBUU6swNO5dPDTWdQlY7DaQq3XWBgISIiqicymQz2KivYq2ofcPK0upLZUnfH4ZjOpjINO6UBx7AeTu0Cjp1SYTqo2Djg+O7MKY+Sr+72SkkGHAYWIiIikchkMjiorOCgsqryxpdllQacsrOkMvLKfm866PhOrhZanR75Wh3yMwqQnFG7gGOvVJjMoCrtmnrxwTZws1fea7PrhIGFiIjIQpQNOP7u9jXuLwgCcjXFZWZMVT4OpzTsZOQZAk6eVoe8SgLO+PsDGqhlNWNgISIiaqJkMhkcbazhaGNd64CToymu0EVVGmpc7cS5ugIwsBAREVEJmUwGJxtrONlYI8Cj5oDTmORiF0BERERUEwYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSPAYWIiIikjwGFiIiIpI8BhYiIiKSvCZzt2ZBEAAAarVa5EqIiIiotko/t0s/x6vSZAJLTk4OAMDPz0/kSoiIiMhcOTk5cHZ2rvJ5mVBTpLEQer0eN2/ehKOjI2QyWb2dV61Ww8/PD8nJyXBycqq380pJU28j22f5mnob2T7L19Tb2JDtEwQBOTk58PHxgVxe9UiVJnOFRS6Xw9fXt8HO7+Tk1CT/IyyrqbeR7bN8Tb2NbJ/la+ptbKj2VXdlpRQH3RIREZHkMbAQERGR5DGw1EClUmH+/PlQqVRil9Jgmnob2T7L19TbyPZZvqbeRim0r8kMuiUiIqKmi1dYiIiISPIYWIiIiEjyGFiIiIhI8hhYiIiISPIYWIiIiEjyGFgArFy5EgEBAbCxsUFERARiYmKq3f/bb7/FfffdBxsbG3Tt2hU7d+5spErrxpz2rV+/HjKZzORhY2PTiNWa5/fff8fjjz8OHx8fyGQy/PjjjzUec/DgQfTo0QMqlQrt2rXD+vXrG7zOe2FuGw8ePFjhPZTJZEhNTW2cgs20ZMkS9OzZE46OjvD09MTQoUORmJhY43GW8ndYl/ZZ0t/hZ599hm7duhlXQO3Vqxd27dpV7TGW8t6VMreNlvT+Vebdd9+FTCbDjBkzqt2vsd/HZh9Ytm7dipkzZ2L+/PmIj49HcHAwBg4ciPT09Er3/+uvvzBy5EhMmDABx48fx9ChQzF06FCcPn26kSuvHXPbBxiWXk5JSTE+kpKSGrFi8+Tl5SE4OBgrV66s1f5XrlzB4MGD8e9//xsnTpzAjBkzMHHiROzZs6eBK607c9tYKjEx0eR99PT0bKAK782hQ4cwefJk/P3339i7dy+Kiorw8MMPIy8vr8pjLOnvsC7tAyzn79DX1xfvvvsu4uLiEBsbi//7v//DkCFDcObMmUr3t6T3rpS5bQQs5/0r79ixY1i9ejW6detW7X6ivI9CMxceHi5MnjzZ+LNOpxN8fHyEJUuWVLr/s88+KwwePNhkW0REhPDSSy81aJ11ZW77oqOjBWdn50aqrn4BEH744Ydq93njjTeEzp07m2wbPny4MHDgwAasrP7Upo0HDhwQAAiZmZmNUlN9S09PFwAIhw4dqnIfS/s7LKs27bPkv0NBEARXV1dhzZo1lT5nye9dWdW10VLfv5ycHCEoKEjYu3ev0LdvX2H69OlV7ivG+9isr7BotVrExcWhf//+xm1yuRz9+/fHkSNHKj3myJEjJvsDwMCBA6vcX0x1aR8A5Obmwt/fH35+fjX+K8LSWNL7d69CQkLg7e2NAQMG4PDhw2KXU2vZ2dkAADc3tyr3seT3sTbtAyzz71Cn02HLli3Iy8tDr169Kt3Hkt87oHZtBCzz/Zs8eTIGDx5c4f2pjBjvY7MOLLdv34ZOp4OXl5fJdi8vryr7+1NTU83aX0x1aV+HDh2wbt06/PTTT9i4cSP0ej169+6N69evN0bJDa6q90+tVqOgoECkquqXt7c3Vq1ahe+++w7fffcd/Pz80K9fP8THx4tdWo30ej1mzJiBPn36oEuXLlXuZ0l/h2XVtn2W9nd46tQpODg4QKVS4eWXX8YPP/yATp06Vbqvpb535rTR0t4/ANiyZQvi4+OxZMmSWu0vxvto1WBnJovUq1cvk3819O7dGx07dsTq1avxzjvviFgZ1VaHDh3QoUMH48+9e/fGpUuX8NFHH+Grr74SsbKaTZ48GadPn8aff/4pdikNorbts7S/ww4dOuDEiRPIzs7Gtm3bEBkZiUOHDlX5gW6JzGmjpb1/ycnJmD59Ovbu3SvpwcHNOrB4eHhAoVAgLS3NZHtaWhpatmxZ6TEtW7Y0a38x1aV95VlbW6N79+64ePFiQ5TY6Kp6/5ycnGBraytSVQ0vPDxc8iFgypQp+OWXX/D777/D19e32n0t6e+wlDntK0/qf4dKpRLt2rUDAISGhuLYsWP43//+h9WrV1fY1xLfO8C8NpYn9fcvLi4O6enp6NGjh3GbTqfD77//jhUrVkCj0UChUJgcI8b72Ky7hJRKJUJDQ7Fv3z7jNr1ej3379lXZN9mrVy+T/QFg79691fZliqUu7StPp9Ph1KlT8Pb2bqgyG5UlvX/16cSJE5J9DwVBwJQpU/DDDz9g//79CAwMrPEYS3of69K+8izt71Cv10Oj0VT6nCW9d9Wpro3lSf39e+ihh3Dq1CmcOHHC+AgLC8OoUaNw4sSJCmEFEOl9bLDhvBZiy5YtgkqlEtavXy+cPXtWePHFFwUXFxchNTVVEARBeP7554WoqCjj/ocPHxasrKyEZcuWCefOnRPmz58vWFtbC6dOnRKrCdUyt30LFy4U9uzZI1y6dEmIi4sTRowYIdjY2AhnzpwRqwnVysnJEY4fPy4cP35cACB8+OGHwvHjx4WkpCRBEAQhKipKeP755437X758WbCzsxNmzZolnDt3Tli5cqWgUCiE3bt3i9WEGpnbxo8++kj48ccfhQsXLginTp0Spk+fLsjlcuG3334TqwnVmjRpkuDs7CwcPHhQSElJMT7y8/ON+1jy32Fd2mdJf4dRUVHCoUOHhCtXrgj//POPEBUVJchkMuHXX38VBMGy37tS5rbRkt6/qpSfJSSF97HZBxZBEIRPPvlEaN26taBUKoXw8HDh77//Nj7Xt29fITIy0mT/b775Rmjfvr2gVCqFzp07Czt27Gjkis1jTvtmzJhh3NfLy0sYNGiQEB8fL0LVtVM6hbf8o7RNkZGRQt++fSscExISIiiVSqFNmzZCdHR0o9dtDnPb+N577wlt27YVbGxsBDc3N6Ffv37C/v37xSm+FiprGwCT98WS/w7r0j5L+jscP3684O/vLyiVSqFFixbCQw89ZPwgFwTLfu9KmdtGS3r/qlI+sEjhfZQJgiA03PUbIiIionvXrMewEBERkWVgYCEiIiLJY2AhIiIiyWNgISIiIsljYCEiIiLJY2AhIiIiyWNgISIiIsljYCEiIiLJY2AhIiIiyWNgISIiIsljYCEiIiLJ+3/aYr/FyCwKLAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"History\")\n",
    "plt.plot(loss_history, label=\"Loss history\")\n",
    "plt.plot(train_history, label=\"Train history\")\n",
    "plt.legend(loc=\"upper left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T14:21:54.792436Z",
     "iopub.status.busy": "2023-10-23T14:21:54.791585Z",
     "iopub.status.idle": "2023-10-23T14:21:54.796574Z",
     "shell.execute_reply": "2023-10-23T14:21:54.795581Z",
     "shell.execute_reply.started": "2023-10-23T14:21:54.792402Z"
    }
   },
   "outputs": [],
   "source": [
    "test_loader = DataLoader(test_dataset, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T15:39:27.080730Z",
     "iopub.status.busy": "2023-10-23T15:39:27.079912Z",
     "iopub.status.idle": "2023-10-23T15:51:52.690571Z",
     "shell.execute_reply": "2023-10-23T15:51:52.689430Z",
     "shell.execute_reply.started": "2023-10-23T15:39:27.080695Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.723790438018895"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = evaluate_model(gpt, test_loader)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T14:08:47.732978Z",
     "iopub.status.busy": "2023-10-23T14:08:47.732612Z",
     "iopub.status.idle": "2023-10-23T14:08:48.543896Z",
     "shell.execute_reply": "2023-10-23T14:08:48.543124Z",
     "shell.execute_reply.started": "2023-10-23T14:08:47.732949Z"
    }
   },
   "outputs": [],
   "source": [
    "torch.save(gpt, 'st_sberbank-gpt-sentiment-classifier.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T15:17:09.532169Z",
     "iopub.status.busy": "2023-10-23T15:17:09.531756Z",
     "iopub.status.idle": "2023-10-23T15:17:10.021491Z",
     "shell.execute_reply": "2023-10-23T15:17:10.020491Z",
     "shell.execute_reply.started": "2023-10-23T15:17:09.532135Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPT2LMHeadModel(\n",
       "  (transformer): GPT2Model(\n",
       "    (wte): Embedding(50264, 768)\n",
       "    (wpe): Embedding(2048, 768)\n",
       "    (drop): Dropout(p=0.1, inplace=False)\n",
       "    (h): ModuleList(\n",
       "      (0-11): 12 x GPT2Block(\n",
       "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): GPT2Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GPT2MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (act): NewGELUActivation()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_model = torch.load('st_sberbank-gpt-sentiment-classifier.pth').to(device)\n",
    "loaded_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-23T15:53:55.960791Z",
     "iopub.status.busy": "2023-10-23T15:53:55.960415Z",
     "iopub.status.idle": "2023-10-23T16:06:24.353850Z",
     "shell.execute_reply": "2023-10-23T16:06:24.352909Z",
     "shell.execute_reply.started": "2023-10-23T15:53:55.960762Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.723790438018895"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = evaluate_model(loaded_model, test_loader)\n",
    "accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
